From c8c95be43e24e5600f06b1803e69148a043e4ec6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 27 Jul 2023 18:54:11 +0530 Subject: [PATCH 01/63] server,api,ui: snapshot copy and zone selection Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/event/EventTypes.java | 3 +- .../storage/snapshot/SnapshotApiService.java | 6 +- .../user/snapshot/CopySnapshotCmd.java | 168 ++++++++++++++++++ .../api/response/SnapshotResponse.java | 8 + .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../storage/snapshot/SnapshotManagerImpl.java | 6 + ui/public/locales/en.json | 3 + ui/src/config/section/storage.js | 3 + ui/src/views/storage/FormSchedule.vue | 56 +++++- .../views/storage/RecurringSnapshotVolume.vue | 1 + ui/src/views/storage/TakeSnapshot.vue | 82 ++++++--- 11 files changed, 313 insertions(+), 24 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 484dc9b001e7..67b6ce3f54c7 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.ha.HAConfig; import org.apache.cloudstack.usage.Usage; +import org.apache.cloudstack.vm.schedule.VMSchedule; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterGuestIpv6Prefix; @@ -84,7 +85,6 @@ import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.VirtualMachine; -import org.apache.cloudstack.vm.schedule.VMSchedule; public class EventTypes { @@ -320,6 +320,7 @@ public class EventTypes { public static final String EVENT_DOMAIN_UPDATE = "DOMAIN.UPDATE"; // Snapshots + public static final String EVENT_SNAPSHOT_COPY = "SNAPSHOT.COPY"; public static final String EVENT_SNAPSHOT_CREATE = "SNAPSHOT.CREATE"; public static final String EVENT_SNAPSHOT_ON_PRIMARY = "SNAPSHOT.ON_PRIMARY"; public static final String EVENT_SNAPSHOT_OFF_PRIMARY = "SNAPSHOT.OFF_PRIMARY"; diff --git a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java index 38e5e105a483..ed47b593f584 100644 --- a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java +++ b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java @@ -18,18 +18,20 @@ import java.util.List; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; +import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; import com.cloud.api.commands.ListRecurringSnapshotScheduleCmd; import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import com.cloud.user.Account; import com.cloud.utils.Pair; -import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; public interface SnapshotApiService { @@ -124,4 +126,6 @@ Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapsh SnapshotPolicy updateSnapshotPolicy(UpdateSnapshotPolicyCmd updateSnapshotPolicyCmd); void markVolumeSnapshotsAsDestroyed(Volume volume); + + Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java new file mode 100644 index 000000000000..82be002b182a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.snapshot; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.dc.DataCenter; +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.Snapshot; +import com.cloud.user.Account; + +@APICommand(name = "copySnapshot", description = "Copies a snapshot from one zone to another.", responseObject = SnapshotResponse.class, responseView = ResponseObject.ResponseView.Restricted, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CopySnapshotCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = SnapshotResponse.class, required = true, description = "the ID of the snapshot.") + private Long id; + + @Parameter(name = ApiConstants.DESTINATION_ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + required = false, + description = "the ID of the zone the snapshot is being copied to.") + protected Long destZoneId; + + @Parameter(name = ApiConstants.DESTINATION_ZONE_ID_LIST, + type=CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ZoneResponse.class, + required = false, + description = "A list of IDs of the zones that the snapshot needs to be copied to." + + "Specify this list if the snapshot needs to copied to multiple zones in one go. " + + "Do not specify destzoneid and destzoneids together, however one of them is required.") + protected List destZoneIds; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public Long getId() { + return id; + } + + public List getDestinationZoneIds() { + if (destZoneIds != null && destZoneIds.size() != 0) { + return destZoneIds; + } + if (destZoneId != null) { + List < Long > destIds = new ArrayList<>(); + destIds.add(destZoneId); + return destIds; + } + return null; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_SNAPSHOT_COPY; + } + + @Override + public String getEventDescription() { + StringBuilder descBuilder = new StringBuilder(); + if (getDestinationZoneIds() != null) { + for (Long destId : getDestinationZoneIds()) { + descBuilder.append(", "); + descBuilder.append(this._uuidMgr.getUuid(DataCenter.class, destId)); + } + if (descBuilder.length() > 0) { + descBuilder.deleteCharAt(0); + } + } + + return "copying snapshot: " + this._uuidMgr.getUuid(Snapshot.class, getId()) + ((descBuilder.length() > 0) ? " to zones: " + descBuilder.toString() : ""); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Snapshot; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + @Override + public long getEntityOwnerId() { + Snapshot snapshot = _entityMgr.findById(Snapshot.class, getId()); + if (snapshot != null) { + return snapshot.getAccountId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() throws ResourceUnavailableException { + try { + if (destZoneId == null && (destZoneIds == null || destZoneIds.size() == 0)) + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Either destzoneid or destzoneids parameters have to be specified."); + + if (destZoneId != null && destZoneIds != null && destZoneIds.size() != 0) + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Both destzoneid and destzoneids cannot be specified at the same time."); + + CallContext.current().setEventDetails(getEventDescription()); + Snapshot snapshot = _snapshotService.copySnapshot(this); + + if (snapshot != null){ + List listResponse = new ArrayList<>();/*_responseGenerator.createSnapshotResponse(getResponseView(), + snapshot, getDestinationZoneIds(), false);*/ + SnapshotResponse response = new SnapshotResponse(); + if (listResponse != null && !listResponse.isEmpty()) { + response = listResponse.get(0); + } + + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to copy snapshot"); + } + } catch (StorageUnavailableException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } + + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index 5490e8a4046d..6e8d4d44aee0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -98,6 +98,10 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements @Param(description = "id of the availability zone") private String zoneId; + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "name of the availability zone") + private String zoneName; + @SerializedName(ApiConstants.REVERTABLE) @Param(description = "indicates whether the underlying storage supports reverting the volume to this snapshot") private boolean revertable; @@ -208,6 +212,10 @@ public void setZoneId(String zoneId) { this.zoneId = zoneId; } + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + public void setTags(Set tags) { this.tags = tags; } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 517ee8dd71cc..78f1b0e317fa 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -641,6 +641,7 @@ public SnapshotResponse createSnapshotResponse(Snapshot snapshot) { DataCenter zone = ApiDBUtils.findZoneById(volume.getDataCenterId()); if (zone != null) { snapshotResponse.setZoneId(zone.getUuid()); + snapshotResponse.setZoneName(zone.getName()); } if (volume.getVolumeType() == Volume.Type.ROOT && volume.getInstanceId() != null) { diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index bd8811b2a157..fd53312c1ac7 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; @@ -1568,4 +1569,9 @@ public void markVolumeSnapshotsAsDestroyed(Volume volume) { } } } + + @Override + public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException { + return null; + } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 2f6f335de7d5..43fa10b54582 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -51,12 +51,14 @@ "label.action.bulk.delete.isos": "Bulk delete ISOs", "label.action.bulk.delete.load.balancer.rules": "Bulk delete load balancer rules", "label.action.bulk.delete.portforward.rules": "Bulk delete port forward rules", +"label.action.bulk.delete.snapshots": "Bulk delete snapshots", "label.action.bulk.delete.templates": "Bulk delete templates", "label.action.bulk.release.public.ip.address": "Bulk release public IP addresses", "label.action.cancel.maintenance.mode": "Cancel maintenance mode", "label.action.change.password": "Change password", "label.action.configure.stickiness": "Stickiness", "label.action.copy.iso": "Copy ISO", +"label.action.copy.snapshot": "Copy snapshot", "label.action.copy.template": "Copy template", "label.action.create.snapshot.from.vmsnapshot": "Create snapshot from VM snapshot", "label.action.create.template.from.volume": "Create template from volume", @@ -485,6 +487,7 @@ "label.confirm.delete.isos": "Please confirm you wish to delete the selected ISOs.", "label.confirm.delete.loadbalancer.rules": "Please confirm you wish to delete the selected load balancing rules.", "label.confirm.delete.portforward.rules": "Please confirm you wish to delete the selected port-forward rules.", +"label.confirm.delete.snapshot.zones": "Please confirm you wish to delete the snapshot in the selected zones.", "label.confirm.delete.templates": "Please confirm you wish to delete the selected templates.", "label.confirm.delete.tungsten.address.group": "Please confirm that you would like to delete this Address Group", "label.confirm.delete.tungsten.firewall.policy": "Please confirm that you would like to delete this Firewall Policy", diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index bec54043a84f..5cfe8598e216 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -322,6 +322,9 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) }, { + name: 'zones', + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/SnapshotZones.vue'))) + }, { name: 'comments', component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) } diff --git a/ui/src/views/storage/FormSchedule.vue b/ui/src/views/storage/FormSchedule.vue index c689de2849d4..c7cc56b5d26c 100644 --- a/ui/src/views/storage/FormSchedule.vue +++ b/ui/src/views/storage/FormSchedule.vue @@ -138,6 +138,37 @@ + + + + + + + + + + + + {{ opt.name || opt.description }} + + + + +
{{ $t('label.tags') }}
@@ -194,6 +225,7 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import TooltipButton from '@/components/widgets/TooltipButton' +import TooltipLabel from '@/components/widgets/TooltipLabel' import { timeZone } from '@/utils/timezone' import { mixinForm } from '@/utils/mixin' import debounce from 'lodash/debounce' @@ -202,7 +234,8 @@ export default { name: 'FormSchedule', mixins: [mixinForm], components: { - TooltipButton + TooltipButton, + TooltipLabel }, props: { loading: { @@ -216,6 +249,10 @@ export default { resource: { type: Object, required: true + }, + resourceType: { + type: String, + default: null } }, data () { @@ -262,6 +299,23 @@ export default { maxsnaps: [{ required: true, message: this.$t('message.error.required.input') }], timezone: [{ required: true, message: `${this.$t('message.error.select')}` }] }) + if (this.resourceType === 'Volume') { + this.fetchZoneData() + } + }, + fetchZoneData () { + const params = {} + params.showicon = true + this.zoneLoading = true + api('listZones', params).then(json => { + const listZones = json.listzonesresponse.zone + if (listZones) { + this.zones = listZones + this.zones = this.zones.filter(zone => zone.type !== 'Edge' && zone.id !== this.resource.zoneid) + } + }).finally(() => { + this.zoneLoading = false + }) }, fetchTimeZone (value) { this.timeZoneMap = [] diff --git a/ui/src/views/storage/RecurringSnapshotVolume.vue b/ui/src/views/storage/RecurringSnapshotVolume.vue index 7f4bd5de0293..ce929848de43 100644 --- a/ui/src/views/storage/RecurringSnapshotVolume.vue +++ b/ui/src/views/storage/RecurringSnapshotVolume.vue @@ -23,6 +23,7 @@ :loading="loading" :resource="resource" :dataSource="dataSource" + :resourceType="'Volume'" @close-action="closeAction" @refresh="handleRefresh"/> diff --git a/ui/src/views/storage/TakeSnapshot.vue b/ui/src/views/storage/TakeSnapshot.vue index 7e450b52e164..af45cc8b40a5 100644 --- a/ui/src/views/storage/TakeSnapshot.vue +++ b/ui/src/views/storage/TakeSnapshot.vue @@ -31,26 +31,47 @@ layout="vertical" @finish="handleSubmit" > - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + {{ opt.name || opt.description }} + + + + + + + + + +
{{ $t('label.tags') }}
@@ -106,12 +127,14 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import { mixinForm } from '@/utils/mixin' import TooltipButton from '@/components/widgets/TooltipButton' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'TakeSnapshot', mixins: [mixinForm], components: { - TooltipButton + TooltipButton, + TooltipLabel }, props: { loading: { @@ -131,6 +154,8 @@ export default { inputValue: '', inputKey: '', inputVisible: '', + zones: [], + zoneLoading: false, tags: [], dataSource: [] } @@ -142,6 +167,7 @@ export default { this.initForm() this.quiescevm = this.resource.quiescevm this.supportsStorageSnapshot = this.resource.supportsstoragesnapshot + this.fetchZoneData() }, methods: { initForm () { @@ -153,6 +179,20 @@ export default { }) this.rules = reactive({}) }, + fetchZoneData () { + const params = {} + params.showicon = true + this.zoneLoading = true + api('listZones', params).then(json => { + const listZones = json.listzonesresponse.zone + if (listZones) { + this.zones = listZones + this.zones = this.zones.filter(zone => zone.type !== 'Edge' && zone.id !== this.resource.zoneid) + } + }).finally(() => { + this.zoneLoading = false + }) + }, handleSubmit (e) { e.preventDefault() if (this.actionLoading) return From 70648aff2176a510029af004d424ce46b24ca164 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 2 Aug 2023 12:22:32 +0530 Subject: [PATCH 02/63] wip Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/VolumeApiService.java | 5 +- .../storage/snapshot/SnapshotApiService.java | 4 +- .../user/snapshot/CopySnapshotCmd.java | 36 ++-- .../user/snapshot/CreateSnapshotCmd.java | 16 +- .../snapshot/CreateSnapshotPolicyCmd.java | 14 ++ .../com/cloud/agent/transport/Request.java | 23 ++- .../storage/template/TemplateLocation.java | 14 +- .../storage/command/DownloadCommand.java | 15 +- .../storage/to/SnapshotObjectTO.java | 10 ++ .../subsystem/api/storage/SnapshotInfo.java | 2 + .../api/storage/SnapshotService.java | 4 + .../com/cloud/storage/SnapshotZoneVO.java | 118 ++++++++++++++ .../cloud/storage/dao/SnapshotZoneDao.java | 26 +++ .../storage/dao/SnapshotZoneDaoImpl.java | 61 +++++++ .../datastore/db/SnapshotDataStoreDao.java | 7 + .../db/SnapshotDataStoreDaoImpl.java | 33 ++++ .../datastore/db/SnapshotDataStoreVO.java | 46 ++++++ ...s-between-management-and-usage-context.xml | 1 + .../META-INF/db/schema-41810to41900.sql | 23 +++ .../storage/snapshot/SnapshotObject.java | 8 + .../storage/snapshot/SnapshotServiceImpl.java | 97 ++++++++++- .../image/BaseImageStoreDriverImpl.java | 75 ++++++++- scripts/storage/secondary/createvolume.sh | 4 +- .../cloud/server/ManagementServerImpl.java | 2 + .../cloud/storage/VolumeApiServiceImpl.java | 27 ++- .../storage/download/DownloadListener.java | 2 + .../storage/download/DownloadMonitor.java | 2 + .../storage/download/DownloadMonitorImpl.java | 104 +++++++++++- .../storage/snapshot/SnapshotManagerImpl.java | 154 +++++++++++++++++- .../storage/VolumeApiServiceImplTest.java | 2 +- .../storage/template/DownloadManagerImpl.java | 70 ++++---- 31 files changed, 915 insertions(+), 90 deletions(-) create mode 100644 engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java create mode 100644 engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java create mode 100644 engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 8b7565d66edf..09701ca2719b 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -19,9 +19,9 @@ package com.cloud.storage; import java.net.MalformedURLException; +import java.util.List; import java.util.Map; -import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd; @@ -37,6 +37,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.user.Account; +import com.cloud.utils.fsm.NoTransitionException; public interface VolumeApiService { @@ -108,7 +109,7 @@ public interface VolumeApiService { Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) throws ResourceAllocationException; - Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException; + Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List zoneIds) throws ResourceAllocationException; Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume, String customId, long owner, String chainInfo, String name); diff --git a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java index ed47b593f584..f5bc5f9f227d 100644 --- a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java +++ b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java @@ -90,7 +90,7 @@ public interface SnapshotApiService { Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException; - Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot) + Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot, List zoneIds) throws ResourceAllocationException; @@ -127,5 +127,5 @@ Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapsh void markVolumeSnapshotsAsDestroyed(Volume volume); - Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException; + Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java index 82be002b182a..9c01f2496e98 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CopySnapshotCmd.java @@ -32,10 +32,12 @@ import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.dc.DataCenter; import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.Snapshot; @@ -54,6 +56,13 @@ public class CopySnapshotCmd extends BaseAsyncCmd implements UserCmd { entityType = SnapshotResponse.class, required = true, description = "the ID of the snapshot.") private Long id; + @Parameter(name = ApiConstants.SOURCE_ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "ID of the zone in which the snapshot is currently present. " + + "If not specified and zone in which the volume of the snapshot is present will be used.") + private Long sourceZoneId; + @Parameter(name = ApiConstants.DESTINATION_ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, @@ -80,6 +89,10 @@ public Long getId() { return id; } + public Long getSourceZoneId() { + return sourceZoneId; + } + public List getDestinationZoneIds() { if (destZoneIds != null && destZoneIds.size() != 0) { return destZoneIds; @@ -135,25 +148,25 @@ public long getEntityOwnerId() { @Override public void execute() throws ResourceUnavailableException { try { - if (destZoneId == null && (destZoneIds == null || destZoneIds.size() == 0)) + if (destZoneId == null && CollectionUtils.isEmpty(destZoneIds)) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Either destzoneid or destzoneids parameters have to be specified."); - if (destZoneId != null && destZoneIds != null && destZoneIds.size() != 0) + if (destZoneId != null && CollectionUtils.isNotEmpty(destZoneIds)) throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Both destzoneid and destzoneids cannot be specified at the same time."); CallContext.current().setEventDetails(getEventDescription()); Snapshot snapshot = _snapshotService.copySnapshot(this); - if (snapshot != null){ - List listResponse = new ArrayList<>();/*_responseGenerator.createSnapshotResponse(getResponseView(), - snapshot, getDestinationZoneIds(), false);*/ - SnapshotResponse response = new SnapshotResponse(); - if (listResponse != null && !listResponse.isEmpty()) { - response = listResponse.get(0); - } - + if (snapshot != null) { +// List listResponse = new ArrayList<>();_responseGenerator.createSnapshotResponse(getResponseView(), +// snapshot, getDestinationZoneIds(), false); +// SnapshotResponse response = new SnapshotResponse(); +// if (CollectionUtils.isNotEmpty(listResponse)) { +// response = listResponse.get(0); +// } + SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot); response.setResponseName(getCommandName()); setResponseObject(response); } else { @@ -162,6 +175,9 @@ public void execute() throws ResourceUnavailableException { } catch (StorageUnavailableException ex) { s_logger.warn("Exception: ", ex); throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage()); + } catch (ResourceAllocationException ex) { + s_logger.warn("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage()); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index aee72b3dbae5..61640658746f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -18,6 +18,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.api.APICommand; @@ -32,6 +33,7 @@ import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -90,6 +92,14 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") private Map tags; + @Parameter(name = ApiConstants.ZONE_ID_LIST, + type=CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ZoneResponse.class, + description = "A list of IDs of the zones in which the snapshot will be made available." + + "The snapshot will always be made available in the zone in which the volume is present.") + protected List zoneIds; + private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject; // /////////////////////////////////////////////////// @@ -148,6 +158,10 @@ private Long getHostId() { return _snapshotService.getHostIdForSnapshotOperation(volume); } + public List getZoneIds() { + return zoneIds; + } + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// @@ -196,7 +210,7 @@ public ApiCommandResourceType getApiResourceType() { @Override public void create() throws ResourceAllocationException { - Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType()); + Snapshot snapshot = _volumeService.allocSnapshot(getVolumeId(), getPolicyId(), getSnapshotName(), getLocationType(), getZoneIds()); if (snapshot != null) { setEntityId(snapshot.getId()); setEntityUuid(snapshot.getUuid()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java index a3b798405eb0..00bfb9e7e2c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java @@ -18,6 +18,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.cloudstack.acl.RoleType; @@ -30,6 +31,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -75,6 +77,14 @@ public class CreateSnapshotPolicyCmd extends BaseCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") private Map tags; + @Parameter(name = ApiConstants.ZONE_ID_LIST, + type=CommandType.LIST, + collectionType = CommandType.UUID, + entityType = ZoneResponse.class, + description = "A list of IDs of the zones in which the snapshots will be made available." + + "The snapshots will always be made available in the zone in which the volume is present.") + protected List zoneIds; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -107,6 +117,10 @@ public boolean isDisplay() { return display; } + public List getZoneIds() { + return zoneIds; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/core/src/main/java/com/cloud/agent/transport/Request.java b/core/src/main/java/com/cloud/agent/transport/Request.java index 28809341f781..241ccd4bbd8b 100644 --- a/core/src/main/java/com/cloud/agent/transport/Request.java +++ b/core/src/main/java/com/cloud/agent/transport/Request.java @@ -32,12 +32,20 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import com.cloud.utils.HumanReadableJson; - import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.BadCommand; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; +import com.cloud.exception.UnsupportedVersionException; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.HumanReadableJson; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonDeserializationContext; @@ -49,16 +57,6 @@ import com.google.gson.JsonSerializer; import com.google.gson.stream.JsonReader; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.BadCommand; -import com.cloud.agent.api.Command; -import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; -import com.cloud.exception.UnsupportedVersionException; -import com.cloud.serializer.GsonHelper; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; - /** * Request is a simple wrapper around command and answer to add sequencing, * versioning, and flags. Note that the version here represents the changes @@ -253,6 +251,7 @@ public Command[] getCommands() { jsonReader.setLenient(true); _cmds = s_gson.fromJson(jsonReader, (Type)Command[].class); } catch (JsonParseException e) { + s_logger.error("Caught problem while parsing JSON command " + _content, e); _cmds = new Command[] { new BadCommand() }; } catch (RuntimeException e) { s_logger.error("Caught problem with " + _content, e); diff --git a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java index 99360eea72c2..6ff53a0410a9 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateLocation.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateLocation.java @@ -19,26 +19,25 @@ package com.cloud.storage.template; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.Properties; -import java.util.Arrays; - -import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; +import org.apache.log4j.Logger; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.utils.NumbersUtil; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - public class TemplateLocation { private static final Logger s_logger = Logger.getLogger(TemplateLocation.class); public final static String Filename = "template.properties"; @@ -65,6 +64,9 @@ public TemplateLocation(StorageLayer storage, String templatePath) { if (_templatePath.matches(".*" + "volumes" + ".*")) { _file = _storage.getFile(_templatePath + "volume.properties"); _resourceType = ResourceType.VOLUME; + } else if (_templatePath.matches(".*" + "snapshots" + ".*")) { + _file = _storage.getFile(_templatePath + "snapshot.properties"); + _resourceType = ResourceType.SNAPSHOT; } else { _file = _storage.getFile(_templatePath + Filename); } @@ -170,6 +172,8 @@ public TemplateProp getTemplateInfo() { tmplInfo.installPath = _templatePath + _props.getProperty("filename"); // _templatePath endsWith / if (_resourceType == ResourceType.VOLUME) { tmplInfo.installPath = tmplInfo.installPath.substring(tmplInfo.installPath.indexOf("volumes")); + } else if (_resourceType == ResourceType.SNAPSHOT) { + tmplInfo.installPath = tmplInfo.installPath.substring(tmplInfo.installPath.indexOf("snapshots")); } else { tmplInfo.installPath = tmplInfo.installPath.substring(tmplInfo.installPath.indexOf("template")); } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java index 29d737fcce93..4032ac0b6322 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.command; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -33,7 +34,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements InternalIdentity { public static enum ResourceType { - VOLUME, TEMPLATE + VOLUME, TEMPLATE, SNAPSHOT } private boolean hvm; @@ -96,6 +97,18 @@ public DownloadCommand(VolumeObjectTO volume, Long maxDownloadSizeInBytes, Strin resourceType = ResourceType.VOLUME; } + public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, String url) { + super(snapshot.getName(), url, null, snapshot.getAccountId()); + _store = snapshot.getDataStore(); + installPath = snapshot.getPath(); + id = snapshot.getId(); + if (_store instanceof NfsTO) { + setSecUrl(((NfsTO)_store).getUrl()); + } + this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; + this.resourceType = ResourceType.SNAPSHOT; + } + @Override public long getId() { return id; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index c62110b179ec..70cb6d155b04 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -42,6 +42,7 @@ public class SnapshotObjectTO implements DataTO { private boolean quiescevm; private String[] parents; private Long physicalSize = (long) 0; + private long accountId; public SnapshotObjectTO() { @@ -51,6 +52,7 @@ public SnapshotObjectTO() { public SnapshotObjectTO(SnapshotInfo snapshot) { this.path = snapshot.getPath(); this.setId(snapshot.getId()); + this.accountId = snapshot.getAccountId(); VolumeInfo vol = snapshot.getBaseVolume(); if (vol != null) { this.volume = (VolumeObjectTO)vol.getTO(); @@ -168,6 +170,14 @@ public String[] getParents() { return parents; } + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + @Override public String toString() { return new StringBuilder("SnapshotTO[datastore=").append(dataStore).append("|volume=").append(volume).append("|path").append(path).append("]").toString(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index ecc412aa79de..3213484694eb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -57,4 +57,6 @@ public interface SnapshotInfo extends DataObject, Snapshot { void markBackedUp() throws CloudRuntimeException; Snapshot getSnapshotVO(); + + long getAccountId(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java index 053e0cdd1340..802a5a58ef01 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.engine.subsystem.api.storage; +import org.apache.cloudstack.framework.async.AsyncCallFuture; + import com.cloud.storage.Snapshot.Event; public interface SnapshotService { @@ -35,4 +37,6 @@ public interface SnapshotService { void processEventOnSnapshotObject(SnapshotInfo snapshot, Event event); void cleanupOnSnapshotBackupFailure(SnapshotInfo snapshot); + + AsyncCallFuture copySnapshot(SnapshotInfo snapshot, DataStore dataStore); } diff --git a/engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java b/engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java new file mode 100644 index 000000000000..82860defd6de --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/SnapshotZoneVO.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.api.InternalIdentity; + +import com.cloud.utils.db.GenericDao; +import com.cloud.utils.db.GenericDaoBase; + +@Entity +@Table(name = "snapshot_zone_ref") +public class SnapshotZoneVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + Long id; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "snapshot_id") + private long snapshotId; + + @Column(name = GenericDaoBase.CREATED_COLUMN) + private Date created = null; + + @Column(name = "last_updated") + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastUpdated = null; + + @Temporal(value = TemporalType.TIMESTAMP) + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + protected SnapshotZoneVO() { + + } + + public SnapshotZoneVO(long zoneId, long snapshotId, Date lastUpdated) { + this.zoneId = zoneId; + this.snapshotId = snapshotId; + this.lastUpdated = lastUpdated; + } + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public long getZoneId() { + return zoneId; + } + + public void setZoneId(long zoneId) { + this.zoneId = zoneId; + } + + public long getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(long snapshotId) { + this.snapshotId = snapshotId; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Date lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Date getRemoved() { + return removed; + } + +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java new file mode 100644 index 000000000000..a3eeba09159c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage.dao; + +import com.cloud.storage.SnapshotZoneVO; +import com.cloud.utils.db.GenericDao; + +public interface SnapshotZoneDao extends GenericDao { + SnapshotZoneVO findByZoneSnapshot(long zoneId, long templateId); + void addSnapshotToZone(long snapshotId, long zoneId); +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java new file mode 100644 index 000000000000..6ce4b437e59e --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotZoneDaoImpl.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage.dao; + +import java.util.Date; + +import org.apache.log4j.Logger; + +import com.cloud.storage.SnapshotZoneVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class SnapshotZoneDaoImpl extends GenericDaoBase implements SnapshotZoneDao { + public static final Logger s_logger = Logger.getLogger(SnapshotZoneDaoImpl.class.getName()); + protected final SearchBuilder ZoneSnapshotSearch; + + public SnapshotZoneDaoImpl() { + + ZoneSnapshotSearch = createSearchBuilder(); + ZoneSnapshotSearch.and("zone_id", ZoneSnapshotSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + ZoneSnapshotSearch.and("template_id", ZoneSnapshotSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + ZoneSnapshotSearch.done(); + } + + @Override + public SnapshotZoneVO findByZoneSnapshot(long zoneId, long templateId) { + SearchCriteria sc = ZoneSnapshotSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("snapshot_id", templateId); + return findOneBy(sc); + } + + @Override + public void addSnapshotToZone(long snapshotId, long zoneId) { + SnapshotZoneVO snapshotZone = findByZoneSnapshot(zoneId, snapshotId); + if (snapshotZone == null) { + snapshotZone = new SnapshotZoneVO(zoneId, snapshotId, new Date()); + persist(snapshotZone); + } else { + snapshotZone.setRemoved(GenericDaoBase.DATE_TO_NULL); + snapshotZone.setLastUpdated(new Date()); + update(snapshotZone.getId(), snapshotZone); + } + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 2ce15894228d..f70306876147 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; @@ -39,10 +40,14 @@ public interface SnapshotDataStoreDao extends GenericDao listBySnapshot(long snapshotId, DataStoreRole role); + SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role); List listDestroyed(long storeId); @@ -91,4 +96,6 @@ public interface SnapshotDataStoreDao extends GenericDao listReadyByVolumeId(long volumeId); + + List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index 066a36ddff45..a7677147297e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -37,6 +37,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.dao.SnapshotDao; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; @@ -63,6 +64,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase stateSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; + private SearchBuilder storeSnapshotDownloadStatusSearch; protected static final List HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING = List.of(Hypervisor.HypervisorType.XenServer); @@ -123,6 +125,12 @@ public boolean configure(String name, Map params) throws Configu snapshotCreatedSearch.and(CREATED, snapshotCreatedSearch.entity().getCreated(), SearchCriteria.Op.BETWEEN); snapshotCreatedSearch.done(); + storeSnapshotDownloadStatusSearch = createSearchBuilder(); + storeSnapshotDownloadStatusSearch.and("snapshot_id", storeSnapshotDownloadStatusSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ); + storeSnapshotDownloadStatusSearch.and("store_id", storeSnapshotDownloadStatusSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ); + storeSnapshotDownloadStatusSearch.and("downloadState", storeSnapshotDownloadStatusSearch.entity().getDownloadState(), SearchCriteria.Op.IN); + storeSnapshotDownloadStatusSearch.done(); + return true; } @@ -203,6 +211,15 @@ public SnapshotDataStoreVO findByStoreSnapshot(DataStoreRole role, long storeId, return findOneBy(sc); } + @Override + public void removeBySnapshotStore(DataStoreRole role, long storeId, long snapshotId) { + SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + sc.setParameters(STORE_ID, storeId); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STORE_ROLE, role); + remove(sc); + } + @Override public SnapshotDataStoreVO findLatestSnapshotForVolume(Long volumeId, DataStoreRole role) { return findOldestOrLatestSnapshotForVolume(volumeId, role, false); @@ -263,6 +280,13 @@ public SnapshotDataStoreVO findBySnapshot(long snapshotId, DataStoreRole role) { return findOneBy(sc); } + @Override + public List listBySnapshot(long snapshotId, DataStoreRole role) { + SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); + sc.setParameters(STATE, State.Ready); + return listBy(sc); + } + @Override public SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role) { SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); @@ -463,4 +487,13 @@ public List listReadyByVolumeId(long volumeId) { sc.setParameters(STATE, State.Ready); return listBy(sc); } + + @Override + public List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status) { + SearchCriteria sc = storeSnapshotDownloadStatusSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + sc.setParameters("store_id", storeId); + sc.setParameters("downloadState", (Object[])status); + return search(sc, null); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index f36216911b02..ff6dc3fb5f5b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.fsm.StateObject; @@ -95,6 +96,19 @@ public class SnapshotDataStoreVO implements StateObject + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 52c58fef1713..75208bc0a98c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -180,3 +180,26 @@ CREATE TABLE `cloud`.`vm_scheduled_job` ( -- Add support for different cluster types for kubernetes ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `cluster_type` varchar(64) DEFAULT 'CloudManaged' COMMENT 'type of cluster'; ALTER TABLE `cloud`.`kubernetes_cluster` MODIFY COLUMN `kubernetes_version_id` bigint unsigned NULL COMMENT 'the ID of the Kubernetes version of this Kubernetes cluster'; + +-- Add table for snapshot zone reference +CREATE TABLE `cloud`.`snapshot_zone_ref` ( + `id` bigint unsigned NOT NULL auto_increment, + `zone_id` bigint unsigned NOT NULL, + `snapshot_id` bigint unsigned NOT NULL, + `created` DATETIME NOT NULL, + `last_updated` DATETIME, + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY (`id`), + CONSTRAINT `fk_snapshot_zone_ref__zone_id` FOREIGN KEY `fk_snapshot_zone_ref__zone_id` (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, + INDEX `i_snapshot_zone_ref__zone_id`(`zone_id`), + CONSTRAINT `fk_snapshot_zone_ref__snapshot_id` FOREIGN KEY `fk_snapshot_zone_ref__snapshot_id` (`snapshot_id`) REFERENCES `snapshots` (`id`) ON DELETE CASCADE, + INDEX `i_snapshot_zone_ref__snapshot_id`(`snapshot_id`), + INDEX `i_snapshot_zone_ref__removed`(`removed`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + +-- Alter snapshot_store_ref table to add download related fields +ALTER TABLE `cloud`.`snapshot_store_ref` + ADD COLUMN `download_state` varchar(255) DEFAULT NULL COMMENT 'the state of the snapshot download' AFTER `volume_id`, + ADD COLUMN `download_pct` int unsigned DEFAULT NULL COMMENT 'the percentage of the snapshot download completed' AFTER `download_state`, + ADD COLUMN `error_str` varchar(255) DEFAULT NULL COMMENT 'the error message when the snapshot download occurs' AFTER `download_pct`, + ADD COLUMN `local_path` varchar(255) DEFAULT NULL COMMENT 'the path of the snapshot download' AFTER `error_str`; \ No newline at end of file diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 2e45bee94b4f..96c898426bb6 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -64,6 +64,7 @@ public class SnapshotObject implements SnapshotInfo { private DataStore store; private Object payload; private Boolean fullBackup; + private String url; @Inject protected SnapshotDao snapshotDao; @Inject @@ -194,9 +195,16 @@ public long getId() { @Override public String getUri() { + if (url != null) { + return url; + } return snapshot.getUuid(); } + public void setUrl(String url) { + this.url = url; + } + @Override public DataStore getDataStore() { return store; diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 4d106f5c4f6a..58e491f90126 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.snapshot; +import java.io.File; import java.util.List; import java.util.concurrent.ExecutionException; @@ -25,8 +26,11 @@ import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -42,16 +46,20 @@ import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.log4j.Logger; -import com.cloud.storage.CreateSnapshotPayload; +import com.cloud.configuration.Config; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.CreateSnapshotPayload; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -82,6 +90,10 @@ public class SnapshotServiceImpl implements SnapshotService { private SnapshotDetailsDao _snapshotDetailsDao; @Inject VolumeDataFactory volFactory; + @Inject + EndPointSelector epSelector; + @Inject + ConfigurationDao _configDao; static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; @@ -132,6 +144,48 @@ public RevertSnapshotContext(AsyncCompletionCallback callback, SnapshotInfo s } + private String generateCopyUrl(String ipAddress, String dir, String path) { + String hostname = ipAddress; + String scheme = "http"; + boolean _sslCopy = false; + String sslCfg = _configDao.getValue(Config.SecStorageEncryptCopy.toString()); + String _ssvmUrlDomain = _configDao.getValue("secstorage.ssl.cert.domain"); + if (sslCfg != null) { + _sslCopy = Boolean.parseBoolean(sslCfg); + } + if(_sslCopy && (_ssvmUrlDomain == null || _ssvmUrlDomain.isEmpty())){ + s_logger.warn("Empty secondary storage url domain, ignoring SSL"); + _sslCopy = false; + } + if (_sslCopy) { + if(_ssvmUrlDomain.startsWith("*")) { + hostname = ipAddress.replace(".", "-"); + hostname = hostname + _ssvmUrlDomain.substring(1); + } else { + hostname = _ssvmUrlDomain; + } + scheme = "https"; + } + return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; + } + + private String generateCopyUrl(SnapshotInfo srcSnapshot) { + DataStore srcStore = srcSnapshot.getDataStore(); + EndPoint ep = epSelector.select(srcSnapshot); + if (ep == null) { + return null; + } + if (ep.getPublicAddr() == null) { + s_logger.warn("A running secondary storage vm has a null public ip?"); + return null; + } + String adjustedPath = srcSnapshot.getPath(); + if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { + adjustedPath += File.separator + adjustedPath.substring(adjustedPath.lastIndexOf(File.separator) + 1) + ".vmdk"; + } + return generateCopyUrl(ep.getPublicAddr(), ((ImageStoreEntity)srcStore).getMountPoint(), adjustedPath); + } + protected Void createSnapshotAsyncCallback(AsyncCallbackDispatcher callback, CreateSnapshotContext context) { CreateCmdResult result = callback.getResult(); SnapshotObject snapshot = (SnapshotObject)context.snapshot; @@ -608,4 +662,45 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } + @Override + public AsyncCallFuture copySnapshot(SnapshotInfo snapshot, DataStore store) { + // generate a URL from source snapshot ssvm to download to destination data store + String url = generateCopyUrl(snapshot); + if (url == null) { + s_logger.warn("Unable to start/resume copy of template " + snapshot.getName() + " to " + store.getName() + + ", no secondary storage vm in running state in source zone"); + throw new CloudRuntimeException("No secondary VM in running state in source template zone "); + } + + SnapshotObject snapshotForCopy = (SnapshotObject)_snapshotFactory.getSnapshot(snapshot, store); + snapshotForCopy.setUrl(url); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Mark template_store_ref entry as Creating"); + } + AsyncCallFuture future = new AsyncCallFuture(); + DataObject snapshotOnStore = store.create(snapshotForCopy); + ((SnapshotObject)snapshotOnStore).setUrl(url); + snapshotOnStore.processEvent(Event.CreateOnlyRequested); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Invoke datastore driver createAsync to create template on destination store"); + } + try { + CopySnapshotContext context = new CopySnapshotContext<>(null, (SnapshotObject)snapshotOnStore, snapshotForCopy, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().copySnapshotAsyncCallback(null, null)).setContext(context); + store.getDriver().createAsync(store, snapshotOnStore, caller); + } catch (CloudRuntimeException ex) { + // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called + SnapshotDataStoreVO templateStoreVO = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); + if (templateStoreVO != null) { + snapshotForCopy.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + } + SnapshotResult res = new SnapshotResult((SnapshotObject)snapshotOnStore, null); + res.setResult(ex.getMessage()); + future.complete(res); + } + return future; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 3ef9fbc4225e..dd810c58c6d1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -32,13 +32,6 @@ import javax.inject.Inject; -import com.cloud.agent.api.to.NfsTO; -import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Upload; -import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; -import org.apache.log4j.Logger; - import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -53,11 +46,15 @@ import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector; +import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; +import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -68,6 +65,8 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.exception.AgentUnavailableException; @@ -76,10 +75,13 @@ import com.cloud.host.dao.HostDao; import com.cloud.secstorage.CommandExecLogDao; import com.cloud.secstorage.CommandExecLogVO; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.StorageManager; +import com.cloud.storage.Upload; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; @@ -107,6 +109,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject TemplateDataStoreDao _templateStoreDao; @Inject + SnapshotDao snapshotDao; + @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + @Inject EndPointSelector _epSelector; @Inject ConfigurationDao configDao; @@ -192,6 +198,12 @@ public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCal LOGGER.debug("Downloading volume to data store " + dataStore.getId()); } _downloadMonitor.downloadVolumeToStorage(data, caller); + } else if (data.getType() == DataObjectType.SNAPSHOT) { + caller.setCallback(caller.getTarget().createSnapshotAsyncCallback(null, null)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Downloading volume to data store " + dataStore.getId()); + } + _downloadMonitor.downloadSnapshotToStorage(data, caller); } } @@ -313,6 +325,55 @@ protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { + DownloadAnswer answer = callback.getResult(); + DataObject obj = context.data; + DataStore store = obj.getDataStore(); + + SnapshotDataStoreVO snapshotStoreVO = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), obj.getId()); + if (snapshotStoreVO != null) { + if (snapshotStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Snapshot is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); + } + return null; + } + SnapshotDataStoreVO updateBuilder = snapshotDataStoreDao.createForUpdate(); + updateBuilder.setDownloadPercent(answer.getDownloadPct()); + updateBuilder.setDownloadState(answer.getDownloadStatus()); + updateBuilder.setLastUpdated(new Date()); + updateBuilder.setErrorString(answer.getErrorString()); + updateBuilder.setJobId(answer.getJobId()); + updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); + updateBuilder.setInstallPath(answer.getInstallPath()); + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + snapshotDataStoreDao.update(snapshotStoreVO.getId(), updateBuilder); + // update size in snapshot table +// SnapshotVO snapshotUpdater = snapshotDao.createForUpdate(); +// snapshotUpdater.setSize(answer.getTemplateSize()); +// snapshotDao.update(obj.getId(), snapshotUpdater); + } + + AsyncCompletionCallback caller = context.getParentCallback(); + + if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || + answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + CreateCmdResult result = new CreateCmdResult(null, null); + result.setSuccess(false); + result.setResult(answer.getErrorString()); + caller.complete(result); + String msg = "Failed to copy snapshot: " + obj.getUuid() + " with error: " + answer.getErrorString(); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, + (snapshotStoreVO == null ? -1L : 1L), null, msg, msg); // ToDo + LOGGER.error(msg); + } else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + CreateCmdResult result = new CreateCmdResult(null, null); + caller.complete(result); + } + return null; + } + @Override public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { CommandResult result = new CommandResult(); diff --git a/scripts/storage/secondary/createvolume.sh b/scripts/storage/secondary/createvolume.sh index 91370dff710d..e5838aea5f09 100755 --- a/scripts/storage/secondary/createvolume.sh +++ b/scripts/storage/secondary/createvolume.sh @@ -18,8 +18,8 @@ -# $Id: createtmplt.sh 9132 2010-06-04 20:17:43Z manuel $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/storage/secondary/createtmplt.sh $ -# createtmplt.sh -- install a volume +# $Id: createvolume.sh 9132 2010-06-04 20:17:43Z manuel $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/storage/secondary/createvolume.sh $ +# createvolume.sh -- install a volume usage() { printf "Usage: %s: -t -n -f -c -d -h [-u] [-v]\n" $(basename $0) >&2 diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index c16dc4eb2f47..ba8d2859e268 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -467,6 +467,7 @@ import org.apache.cloudstack.api.command.user.securitygroup.RevokeSecurityGroupIngressCmd; import org.apache.cloudstack.api.command.user.securitygroup.UpdateSecurityGroupCmd; import org.apache.cloudstack.api.command.user.snapshot.ArchiveSnapshotCmd; +import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotFromVMSnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; @@ -3630,6 +3631,7 @@ public List> getCommands() { cmdList.add(UpdateSecurityGroupCmd.class); cmdList.add(CreateSnapshotCmd.class); cmdList.add(CreateSnapshotFromVMSnapshotCmd.class); + cmdList.add(CopySnapshotCmd.class); cmdList.add(DeleteSnapshotCmd.class); cmdList.add(ArchiveSnapshotCmd.class); cmdList.add(CreateSnapshotPolicyCmd.class); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index bc9a4215fc42..a6bccce77aed 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -34,6 +34,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.InternalIdentity; @@ -3519,7 +3520,7 @@ private boolean isOperationSupported(VMTemplateVO template, UserVmVO userVm) { @Override @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "allocating snapshot", create = true) - public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException { + public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List zoneIds) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); VolumeInfo volume = volFactory.getVolume(volumeId); @@ -3542,7 +3543,6 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, if (ImageFormat.DIR.equals(volume.getFormat())) { throw new InvalidParameterValueException("Snapshot not supported for volume:" + volumeId); } - if (volume.getTemplateId() != null) { VMTemplateVO template = _templateDao.findById(volume.getTemplateId()); Long instanceId = volume.getInstanceId(); @@ -3570,7 +3570,26 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, throw new InvalidParameterValueException("VolumeId: " + volumeId + " please attach this volume to a VM before create snapshot for it"); } - return snapshotMgr.allocSnapshot(volumeId, policyId, snapshotName, locationType, false); + if (CollectionUtils.isNotEmpty(zoneIds)) { + if (Snapshot.LocationType.PRIMARY.equals(locationType)) { + throw new InvalidParameterValueException(String.format("%s cannot be specified with snapshot %s as %s", ApiConstants.ZONE_ID_LIST, ApiConstants.LOCATION_TYPE, Snapshot.LocationType.PRIMARY)); + } + for (Long zoneId : zoneIds) { + DataCenter dataCenter = _dcDao.findById(zoneId); + if (dataCenter == null) { + throw new InvalidParameterValueException("Unable to find the specified zone"); + } + if (Grouping.AllocationState.Disabled.equals(dataCenter.getAllocationState()) && !_accountMgr.isRootAdmin(caller.getId())) { + throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + dataCenter.getName()); + } + if (DataCenter.Type.Edge.equals(dataCenter.getType())) { + throw new InvalidParameterValueException("Snapshot functionality is not supported on zone %s"); + } + } + } + + + return snapshotMgr.allocSnapshot(volumeId, policyId, snapshotName, locationType, false, zoneIds); } @Override @@ -3626,7 +3645,7 @@ public Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName throw new InvalidParameterValueException("Cannot perform this operation, unsupported on storage pool type " + storagePool.getPoolType()); } - return snapshotMgr.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshotName, null, true); + return snapshotMgr.allocSnapshot(volumeId, Snapshot.MANUAL_POLICY_ID, snapshotName, null, true, null); } @Override diff --git a/server/src/main/java/com/cloud/storage/download/DownloadListener.java b/server/src/main/java/com/cloud/storage/download/DownloadListener.java index 9f528195bfa3..7cd2e2a790a6 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadListener.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadListener.java @@ -181,6 +181,8 @@ public void sendCommand(RequestType reqType) { DownloadProgressCommand dcmd = new DownloadProgressCommand(getCommand(), getJobId(), reqType); if (object.getType() == DataObjectType.VOLUME) { dcmd.setResourceType(ResourceType.VOLUME); + } else if (object.getType() == DataObjectType.SNAPSHOT) { + dcmd.setResourceType(ResourceType.SNAPSHOT); } _ssAgent.sendMessageAsync(dcmd, new UploadListener.Callback(_ssAgent.getId(), this)); } catch (Exception e) { diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java index b93c982b51df..028a957ee335 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitor.java @@ -32,4 +32,6 @@ public interface DownloadMonitor extends Manager { public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback callback); + void downloadSnapshotToStorage(DataObject volume, AsyncCompletionCallback callback); + } diff --git a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java index 1954cdea6879..90782dd934b9 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadMonitorImpl.java @@ -25,9 +25,6 @@ import javax.inject.Inject; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -39,17 +36,22 @@ import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.configuration.Config; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; @@ -58,6 +60,7 @@ import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; @Component public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor { @@ -68,6 +71,8 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor @Inject private VolumeDataStoreDao _volumeStoreDao; @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; + @Inject private AgentManager _agentMgr; @Inject private ConfigurationDao _configDao; @@ -115,6 +120,12 @@ public boolean isTemplateUpdateable(Long templateId, Long storeId) { return (downloadsInProgress.size() == 0); } + public boolean isSnapshotUpdateable(Long snapshotId, Long storeId) { + List downloadsInProgress = + snapshotDataStoreDao.listBySnasphotStoreDownloadStatus(snapshotId, storeId, Status.DOWNLOAD_IN_PROGRESS, Status.DOWNLOADED); + return downloadsInProgress.isEmpty(); + } + private void initiateTemplateDownload(DataObject template, AsyncCompletionCallback callback) { boolean downloadJobExists = false; TemplateDataStoreVO vmTemplateStore; @@ -169,6 +180,63 @@ private void initiateTemplateDownload(DataObject template, AsyncCompletionCallba } } + private void initiateSnapshotDownload(DataObject snapshot, AsyncCompletionCallback callback) { + boolean downloadJobExists = false; + DataStore store = snapshot.getDataStore(); + + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); + if (snapshotStore == null) { + snapshotStore = + new SnapshotDataStoreVO(store.getId(), snapshot.getId()); + snapshotStore.setLastUpdated(new Date()); + snapshotStore.setDownloadPercent(0); + snapshotStore.setDownloadState(Status.NOT_DOWNLOADED); + snapshotStore.setLocalDownloadPath(null); + snapshotStore.setErrorString(null); + snapshotStore.setJobId("jobid0000"); + snapshotStore.setRole(store.getRole()); + snapshotStore = snapshotDataStoreDao.persist(snapshotStore); + } else if ((snapshotStore.getJobId() != null) && (snapshotStore.getJobId().length() > 2)) { + downloadJobExists = true; + } + + Long maxSizeInBytes = getMaxSnapshotSizeInBytes(); + if (snapshotStore != null) { + start(); + DownloadCommand dcmd = new DownloadCommand((SnapshotObjectTO)(snapshot.getTO()), maxSizeInBytes, snapshot.getUri()); + dcmd.setProxy(getHttpProxy()); + if (downloadJobExists) { + dcmd = new DownloadProgressCommand(dcmd, snapshotStore.getJobId(), RequestType.GET_OR_RESTART); + dcmd.setResourceType(ResourceType.SNAPSHOT); + } + EndPoint ep = _epSelector.select(snapshot); + if (ep == null) { + String errMsg = "There is no secondary storage VM for downloading snapshot to image store " + store.getName(); + LOGGER.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + DownloadListener dl = new DownloadListener(ep, store, snapshot, _timer, this, dcmd, callback); + ComponentContext.inject(dl); // initialize those auto-wired field in download listener. + if (downloadJobExists) { + // due to handling existing download job issues, we still keep + // downloadState in template_store_ref to avoid big change in + // DownloadListener to use + // new ObjectInDataStore.State transition. TODO: fix this later + // to be able to remove downloadState from template_store_ref. + LOGGER.info("found existing download job"); + dl.setCurrState(snapshotStore.getDownloadState()); + } + + try { + ep.sendMessageAsync(dcmd, new UploadListener.Callback(ep.getId(), dl)); + } catch (Exception e) { + LOGGER.warn("Unable to start /resume download of snapshot " + snapshot.getId() + " to " + store.getName(), e); + dl.setDisconnected(); + dl.scheduleStatusCheck(RequestType.GET_OR_RESTART); + } + } + } + @Override public void downloadTemplateToStorage(DataObject template, AsyncCompletionCallback callback) { if(template != null) { @@ -245,6 +313,26 @@ public void downloadVolumeToStorage(DataObject volume, AsyncCompletionCallback callback) { + long snapshotId = snapshot.getId(); + DataStore store = snapshot.getDataStore(); + if (isSnapshotUpdateable(snapshotId, store.getId())) { + if (snapshot.getUri() != null) { + initiateSnapshotDownload(snapshot, callback); + } else { + LOGGER.info("Snapshot url is null, cannot download"); + DownloadAnswer ans = new DownloadAnswer("Snapshot url is null", Status.UNKNOWN); + callback.complete(ans); + } + } else { + LOGGER.info("Snapshot download is already in progress or already downloaded"); + DownloadAnswer ans = + new DownloadAnswer("Snapshot download is already in progress or already downloaded", Status.UNKNOWN); + callback.complete(ans); + } + } + private Long getMaxTemplateSizeInBytes() { try { return Long.parseLong(_configDao.getValue("max.template.iso.size")) * 1024L * 1024L * 1024L; @@ -261,6 +349,14 @@ private Long getMaxVolumeSizeInBytes() { } } + private Long getMaxSnapshotSizeInBytes() { + try { + return Long.parseLong(_configDao.getValue("storage.max.volume.upload.size")) * 1024L * 1024L * 1024L; + } catch (NumberFormatException e) { + return null; + } + } + private Proxy getHttpProxy() { if (_proxy == null) { return null; diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index fd53312c1ac7..7825598b5127 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -19,16 +19,19 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -46,6 +49,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; @@ -53,6 +57,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -119,6 +124,7 @@ import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.SnapshotScheduleDao; +import com.cloud.storage.dao.SnapshotZoneDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.template.TemplateConstants; @@ -135,6 +141,7 @@ import com.cloud.utils.DateUtil.IntervalType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.Ternary; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; @@ -222,6 +229,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement protected SnapshotHelper snapshotHelper; @Inject DataCenterDao dataCenterDao; + @Inject + SnapshotZoneDao snapshotZoneDao; private int _totalRetries; private int _pauseInterval; @@ -229,6 +238,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement private ScheduledExecutorService backupSnapshotExecutor; + private DataStore getSnapshotZoneImageStore(long snapshotId, long zoneId) { + List zoneStores = dataStoreMgr.getImageStoresByScope(new ZoneScope(zoneId)); + List snapshotImageStoreList = _snapshotStoreDao.listBySnapshot(snapshotId, DataStoreRole.Image); + List snapshotImageStoreIds = snapshotImageStoreList.stream().map(SnapshotDataStoreVO::getDataStoreId).collect(Collectors.toList()); + zoneStores = zoneStores.stream().filter(s -> snapshotImageStoreIds.contains(s.getId())).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(zoneStores)) { + return zoneStores.get(0); + } + return null; + } + protected boolean isBackupSnapshotToSecondaryForZone(long zoneId) { if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) { return false; @@ -1487,11 +1507,11 @@ public void cleanupSnapshotsByVolume(Long volumeId) { @Override public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException { - return allocSnapshot(volumeId, policyId, snapshotName, locationType, false); + return allocSnapshot(volumeId, policyId, snapshotName, locationType, false, null); } @Override - public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot) throws ResourceAllocationException { + public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, Boolean isFromVmSnapshot, List zoneIds) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); VolumeInfo volume = volFactory.getVolume(volumeId); supportedByHypervisor(volume, isFromVmSnapshot); @@ -1571,7 +1591,133 @@ public void markVolumeSnapshotsAsDestroyed(Volume volume) { } @Override - public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException { - return null; + public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException { + Long snapshotId = cmd.getId(); + Long userId = CallContext.current().getCallingUserId(); + Long sourceZoneId = cmd.getSourceZoneId(); + List destZoneIds = cmd.getDestinationZoneIds(); + Account caller = CallContext.current().getCallingAccount(); + + // Verify parameters + SnapshotVO snapshot = _snapshotDao.findById(snapshotId); + if (snapshot == null) { + throw new InvalidParameterValueException("Unable to find snapshot with id"); + } + + // Verify snapshot is backedup and is on Secondary store + if (!Snapshot.State.BackedUp.equals(snapshot.getState())) { + throw new InvalidParameterValueException("Snapshot is not backed up"); + } + if (Snapshot.LocationType.SECONDARY.equals(snapshot.getLocationType())) { + throw new InvalidParameterValueException("Snapshot is not backed up"); + } + if (CollectionUtils.isEmpty(destZoneIds)) { + throw new InvalidParameterValueException("Please specify valid destination zone(s)."); + } + Volume volume = _volsDao.findById(snapshot.getVolumeId()); + if (sourceZoneId == null) { + sourceZoneId = volume.getDataCenterId(); + } + if (destZoneIds.contains(sourceZoneId)) { + throw new InvalidParameterValueException("Please specify different source and destination zones."); + } + DataCenterVO sourceZone = dataCenterDao.findById(sourceZoneId); + if (sourceZone == null) { + throw new InvalidParameterValueException("Please specify a valid source zone."); + } + Map dataCenterVOs = new HashMap<>(); + for (Long destZoneId: destZoneIds) { + DataCenterVO dstZone = dataCenterDao.findById(destZoneId); + if (dstZone == null) { + throw new InvalidParameterValueException("Please specify a valid destination zone."); + } + dataCenterVOs.put(destZoneId, dstZone); + } + + _accountMgr.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, true, snapshot); + + List failedZones = new ArrayList<>(); + + DataStore srcSecStore = getSnapshotZoneImageStore(snapshotId, sourceZoneId); + if (srcSecStore == null) { + throw new InvalidParameterValueException(String.format("There is no snapshot ID: %s ready on image store", snapshot.getUuid())); + } + + for (Long destZoneId : destZoneIds) { + DataStore dstSecStore = getSnapshotZoneImageStore(snapshotId, destZoneId); + if (dstSecStore != null) { + s_logger.debug("There is already snapshot in secondary storage " + dstSecStore.getName() + + " in zone " + destZoneId + " , don't need to copy"); + continue; + } + if (!copy(snapshot, srcSecStore, dataCenterVOs.get(destZoneId))) { + failedZones.add(dataCenterVOs.get(destZoneId).getName()); + } else { + // increase resource count + _resourceLimitMgr.incrementResourceCount(snapshot.getAccountId(), ResourceType.secondary_storage, snapshot.getSize()); + } + } + + if (destZoneIds.size() > failedZones.size()){ + if (!failedZones.isEmpty()) { + s_logger.error("There were failures when copying template to zones: " + + StringUtils.listToCsvTags(failedZones)); + } + return snapshot; + } else { + throw new CloudRuntimeException("Failed to copy template"); + } + } + + @DB + private boolean copy(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { + final long snapshotId = snapshotVO.getId(); + long dstZoneId = dstZone.getId(); + // find all eligible image stores for the destination zone + List dstSecStores = dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(dstZoneId)); + if (CollectionUtils.isEmpty(dstSecStores)) { + throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, dstZone.getId()); + } + AccountVO account = _accountDao.findById(snapshotVO.getAccountId()); + // find the size of the template to be copied + SnapshotDataStoreVO snapshotDataStoreVO = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcSecStore.getId(), snapshotId); + + _resourceLimitMgr.checkResourceLimit(account, ResourceType.template); + _resourceLimitMgr.checkResourceLimit(account, ResourceType.secondary_storage, snapshotDataStoreVO.getSize()); + + // Copy will just find one eligible image store for the destination zone + // and copy snapshot there, not propagate to all image stores + // for that zone + for (DataStore dstSecStore : dstSecStores) { + SnapshotDataStoreVO dstSnapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, dstSecStore.getId(), snapshotId); + if (dstSnapshotStore != null && dstSnapshotStore.getState() == ObjectInDataStoreStateMachine.State.Ready) { + return true; // already downloaded on this image store + } + if (dstSnapshotStore != null && !List.of(ObjectInDataStoreStateMachine.State.Creating, ObjectInDataStoreStateMachine.State.Copying).contains(dstSnapshotStore.getState())) { + _snapshotStoreDao.removeBySnapshotStore(DataStoreRole.Image, snapshotId, dstSecStore.getId()); + } + + SnapshotInfo snapshotOnSecondary = snapshotFactory.getSnapshot(snapshotId, srcSecStore); + AsyncCallFuture future = snapshotSrv.copySnapshot(snapshotOnSecondary, dstSecStore); + try { + SnapshotResult result = future.get(); + if (result.isFailed()) { + s_logger.debug("copy template failed for image store " + dstSecStore.getName() + ":" + result.getResult()); + continue; // try next image store + } + + snapshotZoneDao.addSnapshotToZone(snapshotId, dstZoneId); + + if (account.getId() != Account.ACCOUNT_ID_SYSTEM) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_COPY, account.getId(), dstZoneId, snapshotId, null, null, null, snapshotVO.getSize(), + snapshotVO.getSize(), snapshotVO.getClass().getName(), snapshotVO.getUuid()); + } + return true; + } catch (Exception ex) { + s_logger.debug("failed to copy template to image store:" + dstSecStore.getName() + " ,will try next one"); + } + } + return false; + } } diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 35b69cf82e4c..135a922c082b 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -597,7 +597,7 @@ public void testUpdateMissingRootDiskControllerWithValidChainInfo() { @Test public void testAllocSnapshotNonManagedStorageArchive() { try { - volumeApiServiceImpl.allocSnapshot(6L, 1L, "test", Snapshot.LocationType.SECONDARY); + volumeApiServiceImpl.allocSnapshot(6L, 1L, "test", Snapshot.LocationType.SECONDARY, null); } catch (InvalidParameterValueException e) { Assert.assertEquals(e.getMessage(), "VolumeId: 6 LocationType is supported only for managed storage"); return; diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index e06d7da210db..0d120591ff1f 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.storage.template; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -37,55 +39,53 @@ import javax.naming.ConfigurationException; -import com.cloud.agent.api.to.OVFInformationTO; -import com.cloud.storage.template.Processor; -import com.cloud.storage.template.S3TemplateDownloader; -import com.cloud.storage.template.TemplateDownloader; -import com.cloud.storage.template.TemplateLocation; -import com.cloud.storage.template.MetalinkTemplateDownloader; -import com.cloud.storage.template.HttpTemplateDownloader; -import com.cloud.storage.template.LocalTemplateDownloader; -import com.cloud.storage.template.ScpTemplateDownloader; -import com.cloud.storage.template.TemplateProp; -import com.cloud.storage.template.OVAProcessor; -import com.cloud.storage.template.IsoProcessor; -import com.cloud.storage.template.QCOW2Processor; -import com.cloud.storage.template.VmdkProcessor; -import com.cloud.storage.template.RawImageProcessor; -import com.cloud.storage.template.TARProcessor; -import com.cloud.storage.template.VhdProcessor; -import com.cloud.storage.template.TemplateConstants; +import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; -import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; +import org.apache.cloudstack.utils.security.ChecksumValue; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; -import com.cloud.utils.net.Proxy; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.S3TO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.HttpTemplateDownloader; +import com.cloud.storage.template.IsoProcessor; +import com.cloud.storage.template.LocalTemplateDownloader; +import com.cloud.storage.template.MetalinkTemplateDownloader; +import com.cloud.storage.template.OVAProcessor; +import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; +import com.cloud.storage.template.QCOW2Processor; +import com.cloud.storage.template.RawImageProcessor; +import com.cloud.storage.template.S3TemplateDownloader; +import com.cloud.storage.template.ScpTemplateDownloader; +import com.cloud.storage.template.TARProcessor; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; import com.cloud.storage.template.TemplateDownloader.Status; +import com.cloud.storage.template.TemplateLocation; +import com.cloud.storage.template.TemplateProp; +import com.cloud.storage.template.VhdProcessor; +import com.cloud.storage.template.VmdkProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Proxy; import com.cloud.utils.script.Script; import com.cloud.utils.storage.QCOW2Utils; -import org.apache.cloudstack.utils.security.ChecksumValue; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.commons.lang3.StringUtils; - -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; public class DownloadManagerImpl extends ManagerBase implements DownloadManager { private String _name; @@ -304,8 +304,7 @@ public void setDownloadStatus(String jobId, Status status) { td.setStatus(Status.POST_DOWNLOAD_FINISHED); td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } - } - else { + } else { // For other TemplateDownloaders where files are locally available, // we run the postLocalDownload() method. td.setDownloadError("Download success, starting install "); @@ -376,11 +375,8 @@ private String postRemoteDownload(String jobId) { private String postLocalDownload(String jobId) { DownloadJob dnld = jobs.get(jobId); TemplateDownloader td = dnld.getTemplateDownloader(); - String resourcePath = dnld.getInstallPathPrefix(); // path with mount - // directory - String finalResourcePath = dnld.getTmpltPath(); // template download - // path on secondary - // storage + String resourcePath = dnld.getInstallPathPrefix(); // path with mount directory + String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage ResourceType resourceType = dnld.getResourceType(); File originalTemplate = new File(td.getDownloadLocalPath()); @@ -389,6 +385,9 @@ private String postLocalDownload(String jobId) { LOGGER.info(String.format("No checksum available for '%s'", originalTemplate.getName())); } } + if (ResourceType.SNAPSHOT.equals(resourceType)) { + return "This not implemented yet, so returning failure!"; + } // check or create checksum String checksumErrorMessage = checkOrCreateTheChecksum(dnld, originalTemplate); if (checksumErrorMessage != null) { @@ -409,7 +408,7 @@ private String postLocalDownload(String jobId) { File downloadedTemplate = new File(resourcePath + "/" + templateFilename); _storage.setWorldReadableAndWriteable(downloadedTemplate); - setPermissionsForTheDownloadedTemplate(dnld, resourcePath, resourceType); + setPermissionsForTheDownloadedTemplate(resourcePath, resourceType); TemplateLocation loc = new TemplateLocation(_storage, resourcePath); try { @@ -468,7 +467,10 @@ private String makeTemplatename(String jobId, String extension) { return templateName; } - private void setPermissionsForTheDownloadedTemplate(DownloadJob dnld, String resourcePath, ResourceType resourceType) { + private void setPermissionsForTheDownloadedTemplate(String resourcePath, ResourceType resourceType) { + if (ResourceType.SNAPSHOT.equals(resourceType)) { + return; + } // Set permissions for template/volume.properties String propertiesFile = resourcePath; if (resourceType == ResourceType.TEMPLATE) { From f556cb1967092b74cc07772aa644357d229a0443 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 2 Aug 2023 17:05:21 +0530 Subject: [PATCH 03/63] wip Signed-off-by: Abhishek Kumar --- ui/src/views/storage/SnapshotZones.vue | 539 +++++++++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 ui/src/views/storage/SnapshotZones.vue diff --git a/ui/src/views/storage/SnapshotZones.vue b/ui/src/views/storage/SnapshotZones.vue new file mode 100644 index 000000000000..071d4963b58a --- /dev/null +++ b/ui/src/views/storage/SnapshotZones.vue @@ -0,0 +1,539 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + From de0de2aacd2363279ae0745ac472aa9cb783ed13 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 3 Aug 2023 17:27:25 +0530 Subject: [PATCH 04/63] wip Signed-off-by: Abhishek Kumar --- .../user/snapshot/ListSnapshotsCmd.java | 25 +- .../api/response/SnapshotResponse.java | 2 +- .../apache/cloudstack/query/QueryService.java | 4 + .../StorageSubsystemCommandHandlerBase.java | 17 +- .../PrepareSnapshotZoneCopyAnswer.java | 47 +++ .../PrepareSnapshotZoneCopyCommand.java | 46 +++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-41810to41900.sql | 89 ++++- .../storage/snapshot/SnapshotServiceImpl.java | 47 ++- .../VmwareStorageSubsystemCommandHandler.java | 22 +- .../main/java/com/cloud/api/ApiDBUtils.java | 149 ++++---- .../java/com/cloud/api/ApiResponseHelper.java | 17 + .../com/cloud/api/query/QueryManagerImpl.java | 137 +++++++ .../cloud/api/query/ViewResponseHelper.java | 19 + .../cloud/api/query/dao/SnapshotJoinDao.java | 38 ++ .../api/query/dao/SnapshotJoinDaoImpl.java | 80 +++++ .../cloud/api/query/vo/SnapshotJoinVO.java | 338 ++++++++++++++++++ .../storage/snapshot/SnapshotManagerImpl.java | 29 +- .../storage/template/DownloadManagerImpl.java | 67 +++- 19 files changed, 1071 insertions(+), 103 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java create mode 100644 server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java create mode 100644 server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java create mode 100644 server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java index 0b4a215733cf..80cc681d0b64 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java @@ -65,6 +65,9 @@ public class ListSnapshotsCmd extends BaseListTaggedResourcesCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "list snapshots by zone id") private Long zoneId; + @Parameter(name = ApiConstants.SNAPSHOT, type = CommandType.BOOLEAN, description = "temp parameter") + private boolean newWay; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -93,6 +96,10 @@ public Long getZoneId() { return zoneId; } + public boolean isNewWay() { + return newWay; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -104,15 +111,19 @@ public ApiCommandResourceType getApiResourceType() { @Override public void execute() { - Pair, Integer> result = _snapshotService.listSnapshots(this); ListResponse response = new ListResponse(); - List snapshotResponses = new ArrayList(); - for (Snapshot snapshot : result.first()) { - SnapshotResponse snapshotResponse = _responseGenerator.createSnapshotResponse(snapshot); - snapshotResponse.setObjectName("snapshot"); - snapshotResponses.add(snapshotResponse); + if (isNewWay()) { + response = _queryService.listSnapshots(this); + } else { + Pair, Integer> result = _snapshotService.listSnapshots(this); + List snapshotResponses = new ArrayList(); + for (Snapshot snapshot : result.first()) { + SnapshotResponse snapshotResponse = _responseGenerator.createSnapshotResponse(snapshot); + snapshotResponse.setObjectName("snapshot"); + snapshotResponses.add(snapshotResponse); + } + response.setResponses(snapshotResponses, result.second()); } - response.setResponses(snapshotResponses, result.second()); response.setResponseName(getCommandName()); setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index 6e8d4d44aee0..a6dff90d21a0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -29,7 +29,7 @@ import com.google.gson.annotations.SerializedName; @EntityReference(value = Snapshot.class) -public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse { +public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "ID of the snapshot") private String id; diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 0587294a826f..69d8330519ba 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -44,6 +44,7 @@ import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; @@ -73,6 +74,7 @@ import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -179,4 +181,6 @@ public interface QueryService { ListResponse listManagementServers(ListMgmtsCmd cmd); List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); + + ListResponse listSnapshots(ListSnapshotsCmd cmd); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 4a9a24a9f53d..88dd42a344b2 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -19,23 +19,23 @@ package com.cloud.storage.resource; -import com.cloud.serializer.GsonHelper; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; -import org.apache.log4j.Logger; - import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; +import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyCommand; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -43,6 +43,7 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.serializer.GsonHelper; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Volume; import com.google.gson.Gson; @@ -69,6 +70,8 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) { return execute((AttachCommand)command); } else if (command instanceof DettachCommand) { return execute((DettachCommand)command); + } else if (command instanceof PrepareSnapshotZoneCopyCommand) { + return execute((PrepareSnapshotZoneCopyCommand)command); } else if (command instanceof IntroduceObjectCmd) { return processor.introduceObject((IntroduceObjectCmd)command); } else if (command instanceof SnapshotAndCopyCommand) { @@ -175,6 +178,10 @@ protected Answer execute(DettachCommand cmd) { } } + protected Answer execute(PrepareSnapshotZoneCopyCommand cmd) { + return new PrepareSnapshotZoneCopyAnswer(cmd, "Unsupported command"); + } + private void logCommand(Command cmd) { try { s_logger.debug(String.format("Executing command %s: [%s].", cmd.getClass().getSimpleName(), s_gogger.toJson(cmd))); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java new file mode 100644 index 000000000000..f21211fd3f1a --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.command; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +import com.cloud.agent.api.Answer; + +public class PrepareSnapshotZoneCopyAnswer extends Answer { + private SnapshotObjectTO snapshot; + + public PrepareSnapshotZoneCopyAnswer() { + super(null); + } + + public PrepareSnapshotZoneCopyAnswer(PrepareSnapshotZoneCopyCommand cmd, SnapshotObjectTO snapshot) { + super(cmd); + setSnapshot(snapshot); + } + + public PrepareSnapshotZoneCopyAnswer(PrepareSnapshotZoneCopyCommand cmd, String errMsg) { + super(null, false, errMsg); + } + + public SnapshotObjectTO getSnapshot() { + return snapshot; + } + + public void setSnapshot(SnapshotObjectTO snapshot) { + this.snapshot = snapshot; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java new file mode 100644 index 000000000000..c9c5c7f3377e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.command; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class PrepareSnapshotZoneCopyCommand extends StorageSubSystemCommand { + + private SnapshotObjectTO snapshot; + + public PrepareSnapshotZoneCopyCommand(final SnapshotObjectTO snapshot) { + super(); + this.snapshot = snapshot; + } + + public SnapshotObjectTO getSnapshot() { + return snapshot; + } + + public void setSnapshot(final SnapshotObjectTO snapshot) { + this.snapshot = snapshot; + } + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public void setExecuteInSequence(boolean inSeq) {} +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 04ec733594e2..c00bbc155678 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -176,6 +176,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 75208bc0a98c..616569a515a0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -52,7 +52,7 @@ CREATE VIEW `cloud`.`async_job_view` AS async_job.instance_type = 'Template' or async_job.instance_type = 'Iso' THEN - vm_template.uuid + snapshots.uuid WHEN async_job.instance_type = 'VirtualMachine' or async_job.instance_type = 'ConsoleProxy' @@ -91,7 +91,7 @@ CREATE VIEW `cloud`.`async_job_view` AS left join `cloud`.`volumes` ON async_job.instance_id = volumes.id left join - `cloud`.`vm_template` ON async_job.instance_id = vm_template.id + `cloud`.`snapshots` ON async_job.instance_id = snapshots.id left join `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id left join @@ -202,4 +202,87 @@ ALTER TABLE `cloud`.`snapshot_store_ref` ADD COLUMN `download_state` varchar(255) DEFAULT NULL COMMENT 'the state of the snapshot download' AFTER `volume_id`, ADD COLUMN `download_pct` int unsigned DEFAULT NULL COMMENT 'the percentage of the snapshot download completed' AFTER `download_state`, ADD COLUMN `error_str` varchar(255) DEFAULT NULL COMMENT 'the error message when the snapshot download occurs' AFTER `download_pct`, - ADD COLUMN `local_path` varchar(255) DEFAULT NULL COMMENT 'the path of the snapshot download' AFTER `error_str`; \ No newline at end of file + ADD COLUMN `local_path` varchar(255) DEFAULT NULL COMMENT 'the path of the snapshot download' AFTER `error_str`; + +-- Create snapshot_view +DROP VIEW IF EXISTS `cloud`.`snapshot_view`; +CREATE VIEW `cloud`.`snapshot_view` AS + SELECT + `snapshots`.`id` AS `id`, + `snapshots`.`uuid` AS `uuid`, + `snapshots`.`name` AS `name`, + `snapshots`.`status` AS `status`, + `snapshots`.`disk_offering_id` AS `disk_offering_id`, + `snapshots`.`snapshot_type` AS `snapshot_type`, + `snapshots`.`type_description` AS `type_description`, + `snapshots`.`size` AS `size`, + `snapshots`.`created` AS `created`, + `snapshots`.`removed` AS `removed`, + `snapshots`.`location_type` AS `location_type`, + `snapshots`.`hypervisor_type` AS `hypervisor_type`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `volumes`.`id` AS `volume_id`, + `volumes`.`uuid` AS `volume_uuid`, + `volumes`.`name` AS `volume_name`, + `volumes`.`volume_type` AS `volume_type`, + `volumes`.`size` AS `volume_size`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `snapshot_store_ref`.`store_id` AS `store_id`, + `snapshot_store_ref`.`store_role` AS `store_role`, + `snapshot_store_ref`.`state` AS `store_state`, + `snapshot_store_ref`.`download_state` AS `download_state`, + `snapshot_store_ref`.`download_pct` AS `download_pct`, + `snapshot_store_ref`.`error_str` AS `error_str`, + `snapshot_store_ref`.`size` AS `store_size`, + `snapshot_store_ref`.`created` AS `created_on_store`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`snapshots`.`id`, + '_', + IFNULL(`data_center`.`id`, 0)) AS `snapshot_zone_pair` + FROM + ((((((((((`snapshots` + JOIN `account` ON ((`account`.`id` = `snapshots`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `volumes` ON ((`volumes`.`id` = `snapshots`.`volume_id`))) + LEFT JOIN `snapshot_store_ref` ON ((`snapshot_store_ref`.`snapshot_id` = `snapshots`.`id`))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`snapshot_store_ref`.`store_role` = 'Image') + AND (`snapshot_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `snapshot_store_ref`.`store_id`)))) + LEFT JOIN `storage_pool` ON ((ISNULL(`storage_pool`.`removed`) + AND (`snapshot_store_ref`.`store_role` = 'Primary') + AND (`snapshot_store_ref`.`store_id` IS NOT NULL) + AND (`storage_pool`.`id` = `snapshot_store_ref`.`store_id`)))) + LEFT JOIN `snapshot_zone_ref` ON (((`snapshot_zone_ref`.`snapshot_id` = `snapshots`.`id`) + AND ISNULL(`snapshot_store_ref`.`store_id`) + AND ISNULL(`snapshot_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`storage_pool`.`data_center_id` = `data_center`.`id`) + OR (`snapshot_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `resource_tags` ON ((`resource_tags`.`resource_id` = `snapshots`.`id`) + AND (`resource_tags`.`resource_type` = 'Snapshot'))); \ No newline at end of file diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 58e491f90126..65dea9c4af9d 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.storage.snapshot; -import java.io.File; import java.util.List; import java.util.concurrent.ExecutionException; @@ -55,6 +54,7 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.log4j.Logger; +import com.cloud.agent.api.Answer; import com.cloud.configuration.Config; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; @@ -169,20 +169,32 @@ private String generateCopyUrl(String ipAddress, String dir, String path) { return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; } + private String generateVmwareCopyUrl(SnapshotInfo srcSnapshot, EndPoint ep) { +// if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { +// SnapshotObject snapshotObject = (SnapshotObject) srcSnapshot; +// PrepareSnapshotZoneCopyCommand cmd = new PrepareSnapshotZoneCopyCommand((SnapshotObjectTO) snapshotObject.getTO()); +// ep.sendMessageAsync(cmd); +// } + return null; + } + private String generateCopyUrl(SnapshotInfo srcSnapshot) { - DataStore srcStore = srcSnapshot.getDataStore(); EndPoint ep = epSelector.select(srcSnapshot); if (ep == null) { return null; } + if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { + return generateVmwareCopyUrl(srcSnapshot, ep); + } if (ep.getPublicAddr() == null) { s_logger.warn("A running secondary storage vm has a null public ip?"); return null; } String adjustedPath = srcSnapshot.getPath(); if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { - adjustedPath += File.separator + adjustedPath.substring(adjustedPath.lastIndexOf(File.separator) + 1) + ".vmdk"; + adjustedPath += ".vmdk"; } + DataStore srcStore = srcSnapshot.getDataStore(); return generateCopyUrl(ep.getPublicAddr(), ((ImageStoreEntity)srcStore).getMountPoint(), adjustedPath); } @@ -410,6 +422,31 @@ protected Void copySnapshotAsyncCallback(AsyncCallbackDispatcher callback, CopySnapshotContext context) { + s_logger.info("----------------------executing copySnapshotZoneAsyncCallback"); + CreateCmdResult result = callback.getResult(); + SnapshotInfo destSnapshot = context.destSnapshot; + AsyncCallFuture future = context.future; + SnapshotResult snapResult = new SnapshotResult(destSnapshot, result.getAnswer()); + if (result.isFailed()) { + snapResult.setResult(result.getResult()); + destSnapshot.processEvent(Event.OperationFailed); + future.complete(snapResult); + return null; + } + try { + Answer answer = result.getAnswer(); + destSnapshot.processEvent(Event.OperationSuccessed); + snapResult = new SnapshotResult(_snapshotFactory.getSnapshot(destSnapshot.getId(), destSnapshot.getDataStore()), answer); + future.complete(snapResult); + } catch (Exception e) { + s_logger.debug("Failed to update snapshot state", e); + snapResult.setResult(e.toString()); + future.complete(snapResult); + } + return null; + } + protected Void deleteSnapshotCallback(AsyncCallbackDispatcher callback, DeleteSnapshotContext context) { CommandResult result = callback.getResult(); @@ -687,9 +724,9 @@ public AsyncCallFuture copySnapshot(SnapshotInfo snapshot, DataS s_logger.debug("Invoke datastore driver createAsync to create template on destination store"); } try { - CopySnapshotContext context = new CopySnapshotContext<>(null, (SnapshotObject)snapshotOnStore, snapshotForCopy, future); + CopySnapshotContext context = new CopySnapshotContext<>(null, (SnapshotObject)snapshotOnStore, snapshotForCopy, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); - caller.setCallback(caller.getTarget().copySnapshotAsyncCallback(null, null)).setContext(context); + caller.setCallback(caller.getTarget().copySnapshotZoneAsyncCallback(null, null)).setContext(context); store.getDriver().createAsync(store, snapshotOnStore, caller); } catch (CloudRuntimeException ex) { // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java index 15caa1d878ea..efe76d84f2ae 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java @@ -21,15 +21,15 @@ import java.io.File; import java.util.EnumMap; -import com.cloud.hypervisor.vmware.manager.VmwareManager; -import com.cloud.utils.NumbersUtil; -import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyCommand; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.DataObjectType; @@ -38,9 +38,11 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; +import com.cloud.hypervisor.vmware.manager.VmwareManager; import com.cloud.hypervisor.vmware.manager.VmwareStorageManager; import com.cloud.storage.DataStoreRole; import com.cloud.storage.resource.VmwareStorageProcessor.VmwareStorageProcessorConfigurableFields; +import com.cloud.utils.NumbersUtil; public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemCommandHandlerBase { @@ -202,4 +204,18 @@ protected Answer execute(CopyCommand cmd) { } } + @Override + protected Answer execute(PrepareSnapshotZoneCopyCommand cmd) { + SnapshotObjectTO snapshot = cmd.getSnapshot(); + String parentPath = storageResource.getRootDir(snapshot.getDataStore().getUrl(), _nfsVersion); + String path = snapshot.getPath(); + int index = path.lastIndexOf(File.separator); + String name = path.substring(index + 1); + String snapDir = path.substring(0, index); + int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()), + Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier()); + storageManager.createOva(parentPath + File.separator + snapDir, name, timeout); + snapshot.setPath(snapDir + File.separator + name + ".ova"); + return new PrepareSnapshotZoneCopyAnswer(cmd, snapshot); + } } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 162b1f38addd..837184ac87e3 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -16,6 +16,75 @@ // under the License. package com.cloud.api; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.AffinityGroupResponse; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants.DomainDetails; +import org.apache.cloudstack.api.ApiConstants.HostDetails; +import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BackupOfferingResponse; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.DiskOfferingResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.DomainRouterResponse; +import org.apache.cloudstack.api.response.EventResponse; +import org.apache.cloudstack.api.response.HostForMigrationResponse; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.HostTagResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.NetworkOfferingResponse; +import org.apache.cloudstack.api.response.ProjectAccountResponse; +import org.apache.cloudstack.api.response.ProjectInvitationResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.cloudstack.api.response.ResourceTagResponse; +import org.apache.cloudstack.api.response.SecurityGroupResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.StorageTagResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.VpcOfferingResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.backup.BackupOffering; +import org.apache.cloudstack.backup.BackupSchedule; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; @@ -35,6 +104,7 @@ import com.cloud.api.query.dao.ResourceTagJoinDao; import com.cloud.api.query.dao.SecurityGroupJoinDao; import com.cloud.api.query.dao.ServiceOfferingJoinDao; +import com.cloud.api.query.dao.SnapshotJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.dao.UserAccountJoinDao; @@ -60,6 +130,7 @@ import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.SecurityGroupJoinVO; import com.cloud.api.query.vo.ServiceOfferingJoinVO; +import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -274,72 +345,6 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.acl.Role; -import org.apache.cloudstack.acl.RoleService; -import org.apache.cloudstack.affinity.AffinityGroup; -import org.apache.cloudstack.affinity.AffinityGroupResponse; -import org.apache.cloudstack.affinity.dao.AffinityGroupDao; -import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.cloudstack.api.ApiConstants.DomainDetails; -import org.apache.cloudstack.api.ApiConstants.HostDetails; -import org.apache.cloudstack.api.ApiConstants.VMDetails; -import org.apache.cloudstack.api.ResponseObject.ResponseView; -import org.apache.cloudstack.api.response.AccountResponse; -import org.apache.cloudstack.api.response.AsyncJobResponse; -import org.apache.cloudstack.api.response.BackupOfferingResponse; -import org.apache.cloudstack.api.response.BackupResponse; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.api.response.DiskOfferingResponse; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.DomainRouterResponse; -import org.apache.cloudstack.api.response.EventResponse; -import org.apache.cloudstack.api.response.HostForMigrationResponse; -import org.apache.cloudstack.api.response.HostResponse; -import org.apache.cloudstack.api.response.HostTagResponse; -import org.apache.cloudstack.api.response.ImageStoreResponse; -import org.apache.cloudstack.api.response.InstanceGroupResponse; -import org.apache.cloudstack.api.response.NetworkOfferingResponse; -import org.apache.cloudstack.api.response.ProjectAccountResponse; -import org.apache.cloudstack.api.response.ProjectInvitationResponse; -import org.apache.cloudstack.api.response.ProjectResponse; -import org.apache.cloudstack.api.response.ResourceIconResponse; -import org.apache.cloudstack.api.response.ResourceTagResponse; -import org.apache.cloudstack.api.response.SecurityGroupResponse; -import org.apache.cloudstack.api.response.ServiceOfferingResponse; -import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.api.response.UserResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.api.response.VolumeResponse; -import org.apache.cloudstack.api.response.VpcOfferingResponse; -import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.cloudstack.backup.Backup; -import org.apache.cloudstack.backup.BackupOffering; -import org.apache.cloudstack.backup.BackupSchedule; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; -import org.apache.cloudstack.backup.dao.BackupScheduleDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.AsyncJob; -import org.apache.cloudstack.framework.jobs.AsyncJobManager; -import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; -import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; - -import javax.annotation.PostConstruct; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.Set; public class ApiDBUtils { private static ManagementServer s_ms; @@ -441,6 +446,7 @@ public class ApiDBUtils { static AccountJoinDao s_accountJoinDao; static AsyncJobJoinDao s_jobJoinDao; static TemplateJoinDao s_templateJoinDao; + static SnapshotJoinDao s_snapshotJoinDao; static PhysicalNetworkTrafficTypeDao s_physicalNetworkTrafficTypeDao; static PhysicalNetworkServiceProviderDao s_physicalNetworkServiceProviderDao; @@ -662,6 +668,8 @@ public class ApiDBUtils { private AsyncJobJoinDao jobJoinDao; @Inject private TemplateJoinDao templateJoinDao; + @Inject + private SnapshotJoinDao snapshotJoinDao; @Inject private PhysicalNetworkTrafficTypeDao physicalNetworkTrafficTypeDao; @@ -820,6 +828,7 @@ void init() { s_accountJoinDao = accountJoinDao; s_jobJoinDao = jobJoinDao; s_templateJoinDao = templateJoinDao; + s_snapshotJoinDao = snapshotJoinDao; s_physicalNetworkTrafficTypeDao = physicalNetworkTrafficTypeDao; s_physicalNetworkServiceProviderDao = physicalNetworkServiceProviderDao; @@ -2080,6 +2089,10 @@ public static TemplateResponse newTemplateResponse(EnumSet detail return s_templateJoinDao.newTemplateResponse(detailsView, view, vr); } + public static SnapshotResponse newSnapshotResponse(ResponseView view, SnapshotJoinVO vr) { + return s_snapshotJoinDao.newSnapshotResponse(view, vr); + } + public static TemplateResponse newIsoResponse(TemplateJoinVO vr) { return s_templateJoinDao.newIsoResponse(vr); } @@ -2088,6 +2101,10 @@ public static TemplateResponse fillTemplateDetails(EnumSet detail return s_templateJoinDao.setTemplateResponse(detailsView, view, vrData, vr); } + public static SnapshotResponse fillSnapshotDetails(ResponseView view, SnapshotResponse vrData, SnapshotJoinVO vr) { + return s_snapshotJoinDao.setSnapshotResponse(view, vrData, vr); + } + public static List newTemplateView(VirtualMachineTemplate vr) { return s_templateJoinDao.newTemplateView(vr); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 78f1b0e317fa..60f04cf24a4b 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -2815,6 +2815,23 @@ private void populateOwner(ControlledEntityResponse response, ControlledEntity o response.setDomainName(domain.getName()); } + private void populateOwner(ControlledViewEntityResponse response, ControlledEntity object) { + Account account = ApiDBUtils.findAccountById(object.getAccountId()); + + if (account.getType() == Account.Type.PROJECT) { + // find the project + Project project = ApiDBUtils.findProjectByProjectAccountId(account.getId()); + response.setProjectId(project.getUuid()); + response.setProjectName(project.getName()); + } else { + response.setAccountName(account.getAccountName()); + } + + Domain domain = ApiDBUtils.findDomainById(object.getDomainId()); + response.setDomainId(domain.getUuid()); + response.setDomainName(domain.getName()); + } + public static void populateOwner(ControlledViewEntityResponse response, ControlledViewEntity object) { if (object.getAccountType() == Account.Type.PROJECT) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 63927f2f9efb..d9162e2c305f 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -80,6 +80,7 @@ import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; @@ -109,6 +110,7 @@ import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -155,6 +157,7 @@ import com.cloud.api.query.dao.ResourceTagJoinDao; import com.cloud.api.query.dao.SecurityGroupJoinDao; import com.cloud.api.query.dao.ServiceOfferingJoinDao; +import com.cloud.api.query.dao.SnapshotJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.dao.UserAccountJoinDao; @@ -179,6 +182,7 @@ import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.SecurityGroupJoinVO; import com.cloud.api.query.vo.ServiceOfferingJoinVO; +import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -228,6 +232,8 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; @@ -236,6 +242,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiServiceImpl; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; @@ -461,6 +468,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ManagementServerHostDao msHostDao; + @Inject + private SnapshotJoinDao snapshotJoinDao; + @Inject EntityManager entityManager; @@ -4447,6 +4457,133 @@ public List listRouterHealthChecks(GetRouterHea return responseGenerator.createHealthCheckResponse(_routerDao.findById(routerId), result); } + @Override + public ListResponse listSnapshots(ListSnapshotsCmd cmd) { + Pair, Integer> result = searchForSnapshotsInternal(cmd); + ListResponse response = new ListResponse<>(); + + ResponseView respView = ResponseView.Full; + + List templateResponses = ViewResponseHelper.createSnapshotResponse(respView, result.first().toArray(new SnapshotJoinVO[result.first().size()])); + response.setResponses(templateResponses, result.second()); + return response; + } + + private Pair, Integer> searchForSnapshotsInternal(ListSnapshotsCmd cmd) { + Long id = cmd.getId(); + Map tags = cmd.getTags(); + Account caller = CallContext.current().getCallingAccount(); + Long volumeId = cmd.getVolumeId(); + String name = cmd.getSnapshotName(); + String keyword = cmd.getKeyword(); + String snapshotTypeStr = cmd.getSnapshotType(); + String intervalTypeStr = cmd.getIntervalType(); + Long zoneId = cmd.getZoneId(); + boolean listAll = cmd.listAll(); + List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); + + SearchBuilder sb = snapshotJoinDao.createSearchBuilder(); + + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotZonePair", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); + + List permittedAccountIds = new ArrayList<>(); + Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + _accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccountIds, domainIdRecursiveListProject, listAll, false); + ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + Long domainId = domainIdRecursiveListProject.first(); + Boolean isRecursive = domainIdRecursiveListProject.second(); + // Verify parameters + if (volumeId != null) { + VolumeVO volume = volumeDao.findById(volumeId); + if (volume != null) { + _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); + } + } + + sb.and("statusNEQ", sb.entity().getStatus(), SearchCriteria.Op.NEQ); //exclude those Destroyed snapshot, not showing on UI + sb.and("volumeId", sb.entity().getVolumeId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); + sb.and("snapshotTypeEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.IN); + sb.and("snapshotTypeNEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.NIN); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + + if (tags != null && !tags.isEmpty()) { + SearchBuilder tagSearch = _resourceTagDao.createSearchBuilder(); + for (int count = 0; count < tags.size(); count++) { + tagSearch.or().op("key" + String.valueOf(count), tagSearch.entity().getKey(), SearchCriteria.Op.EQ); + tagSearch.and("value" + String.valueOf(count), tagSearch.entity().getValue(), SearchCriteria.Op.EQ); + tagSearch.cp(); + } + tagSearch.and("resourceType", tagSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + sb.groupBy(sb.entity().getId()); + sb.join("tagSearch", tagSearch, sb.entity().getId(), tagSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER); + } + + SearchCriteria sc = sb.create(); + _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccountIds, listProjectResourcesCriteria); + + sc.setParameters("statusNEQ", Snapshot.State.Destroyed); + + if (volumeId != null) { + sc.setParameters("volumeId", volumeId); + } + + if (tags != null && !tags.isEmpty()) { + int count = 0; + sc.setJoinParameters("tagSearch", "resourceType", ResourceObjectType.Snapshot.toString()); + for (String key : tags.keySet()) { + sc.setJoinParameters("tagSearch", "key" + String.valueOf(count), key); + sc.setJoinParameters("tagSearch", "value" + String.valueOf(count), tags.get(key)); + count++; + } + } + + if (zoneId != null) { + sc.setParameters("dataCenterId", zoneId); + } + + setIdsListToSearchCriteria(sc, ids); + + if (name != null) { + sc.setParameters("name", name); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (keyword != null) { + SearchCriteria ssc = snapshotJoinDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (snapshotTypeStr != null) { + Snapshot.Type snapshotType = SnapshotVO.getSnapshotType(snapshotTypeStr); + if (snapshotType == null) { + throw new InvalidParameterValueException("Unsupported snapshot type " + snapshotTypeStr); + } + if (snapshotType == Snapshot.Type.RECURRING) { + sc.setParameters("snapshotTypeEQ", Snapshot.Type.HOURLY.ordinal(), Snapshot.Type.DAILY.ordinal(), Snapshot.Type.WEEKLY.ordinal(), Snapshot.Type.MONTHLY.ordinal()); + } else { + sc.setParameters("snapshotTypeEQ", snapshotType.ordinal()); + } + } else if (intervalTypeStr != null && volumeId != null) { + Snapshot.Type type = SnapshotVO.getSnapshotType(intervalTypeStr); + if (type == null) { + throw new InvalidParameterValueException("Unsupported snapshot interval type " + intervalTypeStr); + } + sc.setParameters("snapshotTypeEQ", type.ordinal()); + } else { + // Show only MANUAL and RECURRING snapshot types + sc.setParameters("snapshotTypeNEQ", Snapshot.Type.TEMPLATE.ordinal(), Snapshot.Type.GROUP.ordinal()); + } + + return snapshotJoinDao.searchIncludingRemovedAndCount(sc, searchFilter); + } + @Override public String getConfigComponentName() { return QueryService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index ecfda39972ea..39e09d537073 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -50,6 +50,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -79,6 +80,7 @@ import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.SecurityGroupJoinVO; import com.cloud.api.query.vo.ServiceOfferingJoinVO; +import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -593,6 +595,23 @@ public static List createTemplateResponse(EnumSet(vrDataList.values()); } + public static List createSnapshotResponse(ResponseView view, SnapshotJoinVO... snapshots) { + LinkedHashMap vrDataList = new LinkedHashMap<>(); + for (SnapshotJoinVO vr : snapshots) { + SnapshotResponse vrData = vrDataList.get(vr.getSnapshotZonePair()); + if (vrData == null) { + // first time encountering this snapshot + vrData = ApiDBUtils.newSnapshotResponse(view, vr); + } + else{ + // update tags + vrData = ApiDBUtils.fillSnapshotDetails(view, vrData, vr); + } + vrDataList.put(vr.getSnapshotZonePair(), vrData); + } + return new ArrayList(vrDataList.values()); + } + public static List createTemplateUpdateResponse(ResponseView view, TemplateJoinVO... templates) { Hashtable vrDataList = new Hashtable(); for (TemplateJoinVO vr : templates) { diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java new file mode 100644 index 000000000000..3b101e1f257a --- /dev/null +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.api.query.dao; + +import java.util.List; + +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.SnapshotResponse; + +import com.cloud.api.query.vo.SnapshotJoinVO; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDao; +import com.cloud.utils.db.SearchCriteria; + +public interface SnapshotJoinDao extends GenericDao { + + SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, SnapshotJoinVO snapshotJoinVO); + + SnapshotResponse setSnapshotResponse(ResponseObject.ResponseView view, SnapshotResponse snapsData, SnapshotJoinVO snapshot); + + Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter); +} diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java new file mode 100644 index 000000000000..19af823c55cf --- /dev/null +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.api.query.dao; + +import java.util.List; + +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.log4j.Logger; + +import com.cloud.api.ApiResponseHelper; +import com.cloud.api.query.vo.SnapshotJoinVO; +import com.cloud.storage.Snapshot; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchCriteria; + +public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation implements SnapshotJoinDao { + + public static final Logger s_logger = Logger.getLogger(SnapshotJoinDaoImpl.class); + + @Override + public SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, SnapshotJoinVO snapshot) { + SnapshotResponse snapshotResponse = new SnapshotResponse(); + snapshotResponse.setId(snapshot.getUuid()); + // populate owner. + ApiResponseHelper.populateOwner(snapshotResponse, snapshot); + if (snapshot.getVolumeId() != null) { + snapshotResponse.setVolumeId(snapshot.getVolumeUuid()); + snapshotResponse.setVolumeName(snapshot.getVolumeName()); + snapshotResponse.setVolumeType(snapshot.getVolumeType().name()); + snapshotResponse.setVirtualSize(snapshot.getVolumeSize()); + } + snapshotResponse.setZoneId(snapshot.getDataCenterUuid()); + snapshotResponse.setZoneName(snapshot.getDataCenterName()); + snapshotResponse.setCreated(snapshot.getCreated()); + snapshotResponse.setName(snapshot.getName()); + String intervalType = null; + if (snapshot.getSnapshotType() >= 0 && snapshot.getSnapshotType() < Snapshot.Type.values().length) { + intervalType = Snapshot.Type.values()[snapshot.getSnapshotType()].name(); + } + snapshotResponse.setIntervalType(intervalType); + snapshotResponse.setState(snapshot.getStatus()); + snapshotResponse.setLocationType(snapshot.getLocationType() != null ? snapshot.getLocationType().name() : null); + +// SnapshotInfo snapshotInfo = null; + + snapshotResponse.setPhysicaSize(snapshot.getStoreSize()); + + snapshotResponse.setObjectName("snapshot"); + return snapshotResponse; + } + + @Override + public SnapshotResponse setSnapshotResponse(ResponseObject.ResponseView view, SnapshotResponse snapsData, SnapshotJoinVO snapshot) { + return null; + } + + @Override + public Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter) { + List objects = searchIncludingRemoved(sc, filter, null, false); + Integer count = getCountIncludingRemoved(sc); + return new Pair<>(objects, count); + } +} diff --git a/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java new file mode 100644 index 000000000000..190c586c9771 --- /dev/null +++ b/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java @@ -0,0 +1,338 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.api.query.vo; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Table; + +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Snapshot; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.Volume; +import com.cloud.user.Account; +import com.cloud.utils.db.GenericDao; +import com.google.gson.annotations.Expose; + +@Entity +@Table(name = "snapshot_view") +public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements ControlledViewEntity { + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "status") + @Enumerated(value = EnumType.STRING) + private Snapshot.State status; + + @Column(name = "disk_offering_id") + Long diskOfferingId; + + @Column(name = "snapshot_type") + short snapshotType; + + @Column(name = "type_description") + String typeDescription; + + @Column(name = "size") + long size; + + @Column(name = GenericDao.CREATED_COLUMN) + Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + Date removed; + + @Expose + @Column(name = "location_type", updatable = true, nullable = true) + @Enumerated(value = EnumType.STRING) + private Snapshot.LocationType locationType; + + @Column(name = "hypervisor_type") + @Enumerated(value = EnumType.STRING) + Hypervisor.HypervisorType hypervisorType; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "account_uuid") + private String accountUuid; + + @Column(name = "account_name") + private String accountName = null; + + @Column(name = "account_type") + @Enumerated(value = EnumType.ORDINAL) + private Account.Type accountType; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "domain_uuid") + private String domainUuid; + + @Column(name = "domain_name") + private String domainName = null; + + @Column(name = "domain_path") + private String domainPath = null; + + @Column(name = "project_id") + private long projectId; + + @Column(name = "project_uuid") + private String projectUuid; + + @Column(name = "project_name") + private String projectName; + + @Column(name = "data_center_id") + private long dataCenterId; + + @Column(name = "data_center_uuid") + private String dataCenterUuid; + + @Column(name = "data_center_name") + private String dataCenterName; + + @Column(name = "volume_id") + private Long volumeId; + + @Column(name = "volume_uuid") + private String volumeUuid; + + @Column(name = "volume_name") + private String volumeName; + + @Column(name = "volume_type") + @Enumerated(EnumType.STRING) + Volume.Type volumeType = Volume.Type.UNKNOWN; + + @Column(name = "volume_size") + Long volumeSize; + + @Column(name = "store_id") + private Long storeId; + + @Column(name = "store_role") + private String storeRole; + + @Column(name = "store_state") + @Enumerated(EnumType.STRING) + private ObjectInDataStoreStateMachine storeState; + + @Column(name = "download_state") + @Enumerated(EnumType.STRING) + private VMTemplateStorageResourceAssoc.Status downloadState; + + @Column(name = "download_pct") + private int downloadPercent; + + @Column(name = "error_str") + private String errorString; + + @Column(name = "store_size") + private long storeSize; + + @Column(name = "created_on_store") + private Date createdOnStore = null; + + @Column(name = "snapshot_zone_pair") + private String snapshotZonePair; + + @Override + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public Snapshot.State getStatus() { + return status; + } + + public Long getDiskOfferingId() { + return diskOfferingId; + } + + public short getSnapshotType() { + return snapshotType; + } + + public String getTypeDescription() { + return typeDescription; + } + + public long getSize() { + return size; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public Snapshot.LocationType getLocationType() { + return locationType; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public String getAccountUuid() { + return accountUuid; + } + + @Override + public String getAccountName() { + return accountName; + } + + @Override + public Account.Type getAccountType() { + return accountType; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public String getDomainUuid() { + return domainUuid; + } + + @Override + public String getDomainName() { + return domainName; + } + + @Override + public String getDomainPath() { + return domainPath; + } + + public long getProjectId() { + return projectId; + } + + @Override + public String getProjectUuid() { + return projectUuid; + } + + @Override + public String getProjectName() { + return projectName; + } + + public long getDataCenterId() { + return dataCenterId; + } + + public String getDataCenterUuid() { + return dataCenterUuid; + } + + public String getDataCenterName() { + return dataCenterName; + } + + public Long getVolumeId() { + return volumeId; + } + + public String getVolumeUuid() { + return volumeUuid; + } + + public String getVolumeName() { + return volumeName; + } + + public Volume.Type getVolumeType() { + return volumeType; + } + + public Long getVolumeSize() { + return volumeSize; + } + + public Long getStoreId() { + return storeId; + } + + public String getStoreRole() { + return storeRole; + } + + public ObjectInDataStoreStateMachine getStoreState() { + return storeState; + } + + public VMTemplateStorageResourceAssoc.Status getDownloadState() { + return downloadState; + } + + public int getDownloadPercent() { + return downloadPercent; + } + + public String getErrorString() { + return errorString; + } + + public long getStoreSize() { + return storeSize; + } + + public Date getCreatedOnStore() { + return createdOnStore; + } + + public String getSnapshotZonePair() { + return snapshotZonePair; + } + + @Override + public Class getEntityType() { + return Snapshot.class; + } +} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 7825598b5127..4e1ce3c08f56 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -117,6 +117,7 @@ import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -1669,6 +1670,25 @@ public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableExcep } } + private boolean checkAndProcessSnapshotAlreadyExistInStore(SnapshotVO snapshot, DataStore dstSecStore) { + SnapshotDataStoreVO dstSnapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, dstSecStore.getId(), snapshot.getId()); + if (dstSnapshotStore == null) { + return false; + } + if (dstSnapshotStore.getState() == ObjectInDataStoreStateMachine.State.Ready) { + return true; // already downloaded on this image store + } + if (List.of(VMTemplateStorageResourceAssoc.Status.ABANDONED, + VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR, + VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, + VMTemplateStorageResourceAssoc.Status.UNKNOWN).contains(dstSnapshotStore.getDownloadState()) || + !List.of(ObjectInDataStoreStateMachine.State.Creating, + ObjectInDataStoreStateMachine.State.Copying).contains(dstSnapshotStore.getState())) { + _snapshotStoreDao.removeBySnapshotStore(DataStoreRole.Image, snapshot.getId(), dstSecStore.getId()); + } + return false; + } + @DB private boolean copy(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { final long snapshotId = snapshotVO.getId(); @@ -1689,14 +1709,9 @@ private boolean copy(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO // and copy snapshot there, not propagate to all image stores // for that zone for (DataStore dstSecStore : dstSecStores) { - SnapshotDataStoreVO dstSnapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, dstSecStore.getId(), snapshotId); - if (dstSnapshotStore != null && dstSnapshotStore.getState() == ObjectInDataStoreStateMachine.State.Ready) { - return true; // already downloaded on this image store - } - if (dstSnapshotStore != null && !List.of(ObjectInDataStoreStateMachine.State.Creating, ObjectInDataStoreStateMachine.State.Copying).contains(dstSnapshotStore.getState())) { - _snapshotStoreDao.removeBySnapshotStore(DataStoreRole.Image, snapshotId, dstSecStore.getId()); + if (checkAndProcessSnapshotAlreadyExistInStore(snapshotVO, dstSecStore)) { + return true; } - SnapshotInfo snapshotOnSecondary = snapshotFactory.getSnapshot(snapshotId, srcSecStore); AsyncCallFuture future = snapshotSrv.copySnapshot(snapshotOnSecondary, dstSecStore); try { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 0d120591ff1f..dd222a281111 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -24,6 +24,10 @@ import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -127,8 +131,10 @@ private static class DownloadJob { private final ResourceType resourceType; private OVFInformationTO ovfInformationTO; + private URI url; + public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, ResourceType resourceType) { + String installPathPrefix, ResourceType resourceType, URI url) { super(); this.td = td; this.tmpltName = tmpltName; @@ -140,6 +146,7 @@ public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltNam this.templatesize = 0; this.id = id; this.resourceType = resourceType; + this.url = url; } public String getDescription() { @@ -228,6 +235,14 @@ public OVFInformationTO getOvfInformationTO() { public void setOvfInformationTO(OVFInformationTO ovfInformationTO) { this.ovfInformationTO = ovfInformationTO; } + + public URI getUri() { + return url; + } + + public void setUrl(URI url) { + this.url = url; + } } public static final Logger LOGGER = Logger.getLogger(DownloadManagerImpl.class); @@ -365,6 +380,46 @@ private String postRemoteDownload(String jobId) { return result; } + private String getSnapshotInstallNameFromDownloadUrl(URI uri) { + String name = uri.getPath(); + if (StringUtils.isEmpty(name) || !name.contains("/")) { + return null; + } + String[] items = uri.getPath().split("/"); + name = items[items.length - 1]; + if (!name.contains(".")) { + return name; + } + name = (items.length > 1 ? items[items.length - 2] : name.split(".")[0]) + File.separator + name; + return name; + } + + private String postLocalSnapshotDownload(DownloadJob job, String downloadedFile, String resourcePath, String relativeResourcePath) { + String name = getSnapshotInstallNameFromDownloadUrl(job.getUri()); + if (StringUtils.isEmpty(name)) { + name = UUID.randomUUID().toString(); + LOGGER.warn(String.format("Unable to retrieve install filename for snapshot download %s, using a random UUID", downloadedFile)); + } + Path srcPath = Paths.get(downloadedFile); + Path destPath = Paths.get(resourcePath + File.separator + name); + + try { + LOGGER.debug(String.format("Trying to create missing directories (if any) to move snapshot %s.", destPath)); + Files.createDirectories(destPath.getParent()); + LOGGER.debug(String.format("Trying to move downloaded snapshot [%s] to [%s].", srcPath, destPath)); + Files.move(srcPath, destPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + LOGGER.warn(String.format("Something is wrong while processing post snapshot download %s", resourcePath), e); + return "Unable process post snapshot download due to " + e.getMessage(); + } + String installedPath = relativeResourcePath + File.separator + name; + if (installedPath.contains(".")) { + installedPath = installedPath.substring(0, installedPath.lastIndexOf(File.separator)); + } + job.setTmpltPath(installedPath); + return null; + } + /** * Post local download activity (install and cleanup). Executed in context of * downloader thread @@ -379,15 +434,15 @@ private String postLocalDownload(String jobId) { String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage ResourceType resourceType = dnld.getResourceType(); + if (ResourceType.SNAPSHOT.equals(resourceType)) { + return postLocalSnapshotDownload(dnld, td.getDownloadLocalPath(), resourcePath, finalResourcePath); + } File originalTemplate = new File(td.getDownloadLocalPath()); if(StringUtils.isBlank(dnld.getChecksum())) { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("No checksum available for '%s'", originalTemplate.getName())); } } - if (ResourceType.SNAPSHOT.equals(resourceType)) { - return "This not implemented yet, so returning failure!"; - } // check or create checksum String checksumErrorMessage = checkOrCreateTheChecksum(dnld, originalTemplate); if (checksumErrorMessage != null) { @@ -572,7 +627,7 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag } else { throw new CloudRuntimeException("Unable to download from URL: " + url); } - DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType, uri); dj.setTmpltPath(installPathPrefix); jobs.put(jobId, dj); threadPool.execute(td); @@ -639,7 +694,7 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm // including mount directory // on ssvm, while templatePath is the final relative path on // secondary storage. - DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType, uri); dj.setTmpltPath(templatePath); jobs.put(jobId, dj); threadPool.execute(td); From 6328b08d6c16d6f2abc2de393e2aa2f7fec5fed4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 3 Aug 2023 17:55:14 +0530 Subject: [PATCH 05/63] fix Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/api/query/QueryManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index d9162e2c305f..891520b3d7ba 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -4482,8 +4482,6 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps boolean listAll = cmd.listAll(); List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); - SearchBuilder sb = snapshotJoinDao.createSearchBuilder(); - Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotZonePair", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); List permittedAccountIds = new ArrayList<>(); @@ -4500,6 +4498,8 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps } } + SearchBuilder sb = snapshotJoinDao.createSearchBuilder(); + _accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccountIds, listProjectResourcesCriteria); sb.and("statusNEQ", sb.entity().getStatus(), SearchCriteria.Op.NEQ); //exclude those Destroyed snapshot, not showing on UI sb.and("volumeId", sb.entity().getVolumeId(), SearchCriteria.Op.EQ); sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); From 3630bfdd2d1fd3fc3c8ad629c71911b95f015825 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 16 Aug 2023 14:30:04 +0530 Subject: [PATCH 06/63] changes Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/VolumeApiService.java | 2 +- .../apache/cloudstack/api/ApiConstants.java | 1 + .../user/snapshot/CreateSnapshotCmd.java | 2 +- .../user/snapshot/DeleteSnapshotCmd.java | 9 + .../user/snapshot/ListSnapshotsCmd.java | 26 +- .../api/response/SnapshotResponse.java | 8 + .../command/test/CreateSnapshotCmdTest.java | 4 +- .../StorageSubsystemCommandHandlerBase.java | 12 +- .../SimpleHttpMultiFileDownloader.java | 446 ++++++++++++++++++ ....java => QuerySnapshotZoneCopyAnswer.java} | 22 +- ...java => QuerySnapshotZoneCopyCommand.java} | 8 +- .../api/storage/SnapshotDataFactory.java | 6 +- .../api/storage/SnapshotService.java | 5 +- .../cloud/vm/VmWorkTakeVolumeSnapshot.java | 12 +- .../orchestration/StorageOrchestrator.java | 4 +- .../java/com/cloud/storage/SnapshotVO.java | 14 +- .../datastore/db/SnapshotDataStoreDao.java | 8 +- .../db/SnapshotDataStoreDaoImpl.java | 26 +- .../META-INF/db/schema-41810to41900.sql | 10 +- .../image/SecondaryStorageServiceImpl.java | 2 +- .../snapshot/CephSnapshotStrategy.java | 3 +- .../snapshot/DefaultSnapshotStrategy.java | 64 +-- .../snapshot/ScaleIOSnapshotStrategy.java | 2 +- .../snapshot/SnapshotDataFactoryImpl.java | 50 +- .../storage/snapshot/SnapshotObject.java | 4 +- .../storage/snapshot/SnapshotServiceImpl.java | 111 +++-- .../StorageSystemSnapshotStrategy.java | 11 +- .../snapshot/DefaultSnapshotStrategyTest.java | 63 +-- .../snapshot/SnapshotDataFactoryImplTest.java | 4 +- .../ObjectInDataStoreManagerImpl.java | 2 +- .../presetvariables/PresetVariableHelper.java | 4 +- .../PresetVariableHelperTest.java | 12 +- .../vmware/resource/VmwareResource.java | 1 + .../VmwareStorageSubsystemCommandHandler.java | 37 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 18 +- .../StorPoolPrimaryDataStoreDriver.java | 91 ++-- .../snapshot/StorPoolSnapshotStrategy.java | 2 +- .../java/com/cloud/api/ApiResponseHelper.java | 4 +- .../com/cloud/api/query/QueryManagerImpl.java | 43 +- .../cloud/api/query/ViewResponseHelper.java | 4 +- .../cloud/api/query/dao/SnapshotJoinDao.java | 5 +- .../api/query/dao/SnapshotJoinDaoImpl.java | 110 ++++- .../cloud/api/query/vo/SnapshotJoinVO.java | 26 +- .../cloud/storage/CreateSnapshotPayload.java | 11 + .../com/cloud/storage/StorageManagerImpl.java | 2 +- .../cloud/storage/VolumeApiServiceImpl.java | 64 ++- .../storage/snapshot/SnapshotManager.java | 2 - .../storage/snapshot/SnapshotManagerImpl.java | 322 ++++++++----- .../cloudstack/snapshot/SnapshotHelper.java | 24 +- .../storage/VolumeApiServiceImplTest.java | 4 +- .../storage/snapshot/SnapshotManagerTest.java | 22 +- .../resource/NfsSecondaryStorageResource.java | 31 ++ .../storage/template/DownloadManagerImpl.java | 159 +++++-- ui/src/views/storage/FormSchedule.vue | 14 +- ui/src/views/storage/SnapshotZones.vue | 9 +- ui/src/views/storage/TakeSnapshot.vue | 14 +- 56 files changed, 1466 insertions(+), 510 deletions(-) create mode 100644 core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java rename core/src/main/java/org/apache/cloudstack/storage/command/{PrepareSnapshotZoneCopyAnswer.java => QuerySnapshotZoneCopyAnswer.java} (62%) rename core/src/main/java/org/apache/cloudstack/storage/command/{PrepareSnapshotZoneCopyCommand.java => QuerySnapshotZoneCopyCommand.java} (84%) diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 09701ca2719b..224ef28d24f6 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -106,7 +106,7 @@ public interface VolumeApiService { Volume detachVolumeFromVM(DetachVolumeCmd cmd); - Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) + Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags, List zoneIds) throws ResourceAllocationException; Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType, List zoneIds) throws ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3e0e65220e17..de562234d7a4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -492,6 +492,7 @@ public class ApiConstants { public static final String ZONE = "zone"; public static final String ZONE_ID = "zoneid"; public static final String ZONE_NAME = "zonename"; + public static final String ZONE_WISE = "zonewise"; public static final String NETWORK_TYPE = "networktype"; public static final String PAGE = "page"; public static final String PAGE_SIZE = "pagesize"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index 61640658746f..4ee4918c567e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -224,7 +224,7 @@ public void execute() { Snapshot snapshot; try { snapshot = - _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags()); + _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags(), getZoneIds()); if (snapshot != null) { SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java index 8530e0ff5840..c484e1405329 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/DeleteSnapshotCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -48,6 +49,10 @@ public class DeleteSnapshotCmd extends BaseAsyncCmd { @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType = SnapshotResponse.class, required=true, description="The ID of the snapshot") private Long id; + @Parameter(name=ApiConstants.ZONE_ID, type=CommandType.UUID, entityType = ZoneResponse.class, + description="The ID of the zone for the snapshot") + private Long zoneId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -57,6 +62,10 @@ public Long getId() { return id; } + public Long getZoneId() { + return zoneId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java index 80cc681d0b64..5e4dd5c6d9ff 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java @@ -68,6 +68,13 @@ public class ListSnapshotsCmd extends BaseListTaggedResourcesCmd { @Parameter(name = ApiConstants.SNAPSHOT, type = CommandType.BOOLEAN, description = "temp parameter") private boolean newWay; + @Parameter(name = ApiConstants.SHOW_UNIQUE, type = CommandType.BOOLEAN, description = "If set to false, list templates across zones and their storages") + private Boolean showUnique; + + @Parameter(name = ApiConstants.LOCATION_TYPE, type = CommandType.STRING, description = "list snapshots by location type. Used only when showunique=false." + + "Valid location types: 'primary', 'secondary'. Default is empty") + private String locationType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -97,7 +104,24 @@ public Long getZoneId() { } public boolean isNewWay() { - return newWay; + if (Boolean.FALSE.equals(newWay)) { + return false; + } + return true; + } + + public boolean isShowUnique() { + if (Boolean.FALSE.equals(showUnique)) { + return false; + } + return true; + } + + public String getLocationType() { + if (!isShowUnique()) { + return locationType; + } + return null; } ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index a6dff90d21a0..e56b505f64db 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -118,6 +118,10 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements @Param(description = "virtual size of backedup snapshot on image store") private long virtualSize; + @SerializedName(ApiConstants.DATASTORE_TYPE) + @Param(description = "type of the datastore for the snapshot entry") + private String datastoreType; + public SnapshotResponse() { tags = new LinkedHashSet(); } @@ -239,4 +243,8 @@ public void setOsDisplayName(String osDisplayName) { public void setVirtualSize(long virtualSize) { this.virtualSize = virtualSize; } + + public void setDatastoreType(String datastoreType) { + this.datastoreType = datastoreType; + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index 0d3251a64dff..1a5e8809fc48 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -92,7 +92,7 @@ public void testCreateSuccess() { Snapshot snapshot = Mockito.mock(Snapshot.class); try { Mockito.when(volumeApiService.takeSnapshot(nullable(Long.class), nullable(Long.class), isNull(), - nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), nullable(Map.class))).thenReturn(snapshot); + nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), nullable(Map.class), Mockito.anyList())).thenReturn(snapshot); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); @@ -125,7 +125,7 @@ public void testCreateFailure() { try { Mockito.when(volumeApiService.takeSnapshot(nullable(Long.class), nullable(Long.class), nullable(Long.class), - nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), anyObject())).thenReturn(null); + nullable(Account.class), nullable(Boolean.class), nullable(Snapshot.LocationType.class), nullable(Boolean.class), anyObject(), Mockito.anyList())).thenReturn(null); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 88dd42a344b2..75d5f49d4c6c 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -28,8 +28,8 @@ import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; -import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyAnswer; -import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyCommand; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; @@ -70,8 +70,6 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) { return execute((AttachCommand)command); } else if (command instanceof DettachCommand) { return execute((DettachCommand)command); - } else if (command instanceof PrepareSnapshotZoneCopyCommand) { - return execute((PrepareSnapshotZoneCopyCommand)command); } else if (command instanceof IntroduceObjectCmd) { return processor.introduceObject((IntroduceObjectCmd)command); } else if (command instanceof SnapshotAndCopyCommand) { @@ -84,6 +82,8 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) { return processor.checkDataStoreStoragePolicyCompliance((CheckDataStoreStoragePolicyComplainceCommand) command); } else if (command instanceof SyncVolumePathCommand) { return processor.syncVolumePath((SyncVolumePathCommand) command); + } else if (command instanceof QuerySnapshotZoneCopyCommand) { + return execute((QuerySnapshotZoneCopyCommand)command); } return new Answer((Command)command, false, "not implemented yet"); @@ -178,8 +178,8 @@ protected Answer execute(DettachCommand cmd) { } } - protected Answer execute(PrepareSnapshotZoneCopyCommand cmd) { - return new PrepareSnapshotZoneCopyAnswer(cmd, "Unsupported command"); + protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { + return new QuerySnapshotZoneCopyAnswer(cmd, "Unsupported command"); } private void logCommand(Command cmd) { diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java new file mode 100644 index 000000000000..3419378a5fe5 --- /dev/null +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -0,0 +1,446 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.storage.template; + +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpMethodRetryHandler; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.NoHttpResponseException; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import com.cloud.storage.StorageLayer; + +public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { + public static final Logger s_logger = Logger.getLogger(HttpTemplateDownloader.class.getName()); + private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); + + private static final int CHUNK_SIZE = 1024 * 1024; //1M + private String[] downloadUrls; + private String currentToFile; + public TemplateDownloader.Status currentStatus; + public TemplateDownloader.Status status; + private String errorString = null; + private long currentRemoteSize = 0; + public long downloadTime = 0; + public long currentTotalBytes; + public long totalBytes = 0; + private final HttpClient client; + private GetMethod request; + private boolean resume = false; + private DownloadCompleteCallback completionCallback; + StorageLayer _storage; + boolean inited = true; + + private String toDir; + private long maxTemplateSizeInBytes; + private DownloadCommand.ResourceType resourceType = DownloadCommand.ResourceType.TEMPLATE; + private final HttpMethodRetryHandler retryHandler; + + private HashMap urlFileMap; + + public SimpleHttpMultiFileDownloader(StorageLayer storageLayer, String[] downloadUrls, String toDir, + DownloadCompleteCallback callback, long maxTemplateSizeInBytes, + DownloadCommand.ResourceType resourceType) { + _storage = storageLayer; + this.downloadUrls = downloadUrls; + this.toDir = toDir; + this.resourceType = resourceType; + this.maxTemplateSizeInBytes = maxTemplateSizeInBytes; + completionCallback = callback; + status = TemplateDownloader.Status.NOT_STARTED; + currentStatus = TemplateDownloader.Status.NOT_STARTED; + currentTotalBytes = 0; + client = new HttpClient(s_httpClientManager); + retryHandler = createRetryTwiceHandler(); + urlFileMap = new HashMap<>(); + } + + private GetMethod createRequest(String downloadUrl) { + GetMethod request = new GetMethod(downloadUrl); + request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); + request.setFollowRedirects(true); + return request; + } + + private void checkTemporaryDestination(String toDir) { + try { + File f = File.createTempFile("dnld", "tmp_", new File(toDir)); + if (_storage != null) { + _storage.setWorldReadableAndWriteable(f); + } + currentToFile = f.getAbsolutePath(); + } catch (IOException ex) { + errorString = "Unable to start download -- check url? "; + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + s_logger.warn("Exception in constructor -- " + ex.toString()); + } + } + + private HttpMethodRetryHandler createRetryTwiceHandler() { + return new HttpMethodRetryHandler() { + @Override + public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) { + if (executionCount >= 2) { + // Do not retry if over max retry count + return false; + } + if (exception instanceof NoHttpResponseException) { + // Retry if the server dropped connection on us + return true; + } + if (!method.isRequestSent()) { + // Retry if the request has not been sent fully or + // if it's OK to retry methods that have been sent + return true; + } + // otherwise do not retry + return false; + } + }; + } + + private long downloadFile(String downloadUrl) { + s_logger.info("Starting download for " + downloadUrl); + File file = null; + request = null; + try { + request = createRequest(downloadUrl); + checkTemporaryDestination(toDir); + urlFileMap.put(downloadUrl, currentToFile); + file = new File(currentToFile); + long localFileSize = checkLocalFileSizeForResume(resume, file); + if (checkServerResponse(localFileSize)) return 0; + if (!tryAndGetRemoteSize()) return 0; + if (!canHandleDownloadSize()) return 0; + checkAndSetDownloadSize(); + try (InputStream in = request.getResponseBodyAsStream(); + RandomAccessFile out = new RandomAccessFile(file, "rw"); + ) { + out.seek(localFileSize); + s_logger.info("Starting download from " + downloadUrl + " to " + currentToFile + " remoteSize=" + toHumanReadableSize(currentRemoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); + if (copyBytes(file, in, out)) return 0; + checkDowloadCompletion(); + } + return currentTotalBytes; + } catch (HttpException hte) { + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString = hte.getMessage(); + } catch (IOException ioe) { + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? + // Let's not overwrite the original error message. + if (errorString == null) { + errorString = ioe.getMessage(); + } + } finally { + if (currentStatus == Status.UNRECOVERABLE_ERROR && file != null && file.exists() && !file.isDirectory()) { + file.delete(); + } + if (request != null) { + request.releaseConnection(); + } + } + return 0; + } + + @Override + public long download(boolean resume, DownloadCompleteCallback callback) { + if (skipDownloadOnStatus()) return 0; + if (resume) { + s_logger.error("Resume not allowed for this downloader"); + status = Status.UNRECOVERABLE_ERROR; + return 0; + } + s_logger.info("~~~~~Starting downloads"); + status = Status.IN_PROGRESS; + Date start = new Date(); + for (String downloadUrl : downloadUrls) { + if (StringUtils.isBlank(downloadUrl)) { + continue; + } + long bytes = downloadFile(downloadUrl); + if (currentStatus != Status.DOWNLOAD_FINISHED) { + break; + } + totalBytes += bytes; + } + status = currentStatus; + Date finish = new Date(); + downloadTime += finish.getTime() - start.getTime(); + if (callback != null) { + callback.downloadComplete(status); + } + return 0; + } + + private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException { + int bytes; + byte[] block = new byte[CHUNK_SIZE]; + long offset = 0; + boolean done = false; + currentStatus = Status.IN_PROGRESS; + while (!done && currentStatus != Status.ABORTED && offset <= currentRemoteSize) { + if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { + offset = writeBlock(bytes, out, block, offset); + } else { + done = true; + } + } + out.getFD().sync(); + return false; + } + + private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException { + out.write(block, 0, bytes); + offset += bytes; + out.seek(offset); + currentTotalBytes += bytes; + return offset; + } + + private void checkDowloadCompletion() { + String downloaded = "(incomplete download)"; + if (currentTotalBytes >= currentRemoteSize) { + currentStatus = Status.DOWNLOAD_FINISHED; + downloaded = "(download complete remote=" + toHumanReadableSize(currentRemoteSize) + " bytes)"; + } + errorString = "Downloaded " + toHumanReadableSize(currentTotalBytes) + " bytes " + downloaded; + } + + private boolean canHandleDownloadSize() { + if (currentRemoteSize > maxTemplateSizeInBytes) { + s_logger.info("Remote size is too large: " + toHumanReadableSize(currentRemoteSize) + " , max=" + toHumanReadableSize(maxTemplateSizeInBytes)); + currentStatus = Status.UNRECOVERABLE_ERROR; + errorString = "Download file size is too large"; + return false; + } + return true; + } + + private void checkAndSetDownloadSize() { + if (currentRemoteSize == 0) { + currentRemoteSize = maxTemplateSizeInBytes; + } + } + + private boolean tryAndGetRemoteSize() { + Header contentLengthHeader = request.getResponseHeader("content-length"); + boolean chunked = false; + long reportedRemoteSize = 0; + if (contentLengthHeader == null) { + Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); + if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + currentStatus = Status.UNRECOVERABLE_ERROR; + errorString = " Failed to receive length of download "; + return false; + } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + chunked = true; + } + } else { + reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue()); + if (reportedRemoteSize == 0) { + currentStatus = Status.DOWNLOAD_FINISHED; + String downloaded = "(download complete remote=" + currentRemoteSize + "bytes)"; + errorString = "Downloaded " + currentTotalBytes + " bytes " + downloaded; + downloadTime = 0; + return false; + } + } + + if (currentRemoteSize == 0) { + currentRemoteSize = reportedRemoteSize; + } + return true; + } + + private boolean checkServerResponse(long localFileSize) throws IOException { + int responseCode = 0; + + if (localFileSize > 0) { + // require partial content support for resume + request.addRequestHeader("Range", "bytes=" + localFileSize + "-"); + if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) { + errorString = "HTTP Server does not support partial get"; + currentStatus = Status.UNRECOVERABLE_ERROR; + return true; + } + } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { + currentStatus = Status.UNRECOVERABLE_ERROR; + errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; + return true; //FIXME: retry? + } + return false; + } + + private long checkLocalFileSizeForResume(boolean resume, File file) { + // TODO check the status of this downloader as well? + long localFileSize = 0; + if (file.exists() && resume) { + localFileSize = file.length(); + s_logger.info("Resuming download to file (current size)=" + toHumanReadableSize(localFileSize)); + } + return localFileSize; + } + + private boolean skipDownloadOnStatus() { + switch (currentStatus) { + case ABORTED: + case UNRECOVERABLE_ERROR: + case DOWNLOAD_FINISHED: + return true; + default: + + } + return false; + } + + public String[] getDownloadUrls() { + return downloadUrls; + } + + public String getCurrentToFile() { + File file = new File(currentToFile); + + return file.getAbsolutePath(); + } + + @Override + public TemplateDownloader.Status getStatus() { + return currentStatus; + } + + @Override + public long getDownloadTime() { + return downloadTime; + } + + @Override + public long getDownloadedBytes() { + return currentTotalBytes; + } + + @Override + @SuppressWarnings("fallthrough") + public boolean stopDownload() { + switch (getStatus()) { + case IN_PROGRESS: + if (request != null) { + request.abort(); + } + currentStatus = TemplateDownloader.Status.ABORTED; + return true; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + currentStatus = TemplateDownloader.Status.ABORTED; + case DOWNLOAD_FINISHED: + File f = new File(currentToFile); + if (f.exists()) { + f.delete(); + } + return true; + + default: + return true; + } + } + + @Override + public int getDownloadPercent() { + return 0; + } + + @Override + protected void runInContext() { + try { + download(resume, completionCallback); + } catch (Throwable t) { + s_logger.warn("Caught exception during download " + t.getMessage(), t); + errorString = "Failed to install: " + t.getMessage(); + currentStatus = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + } + + } + + @Override + public void setStatus(TemplateDownloader.Status status) { + this.currentStatus = status; + } + + public boolean isResume() { + return resume; + } + + @Override + public String getDownloadError() { + return errorString == null ? " " : errorString; + } + + @Override + public String getDownloadLocalPath() { + return toDir; + } + + @Override + public void setResume(boolean resume) { + this.resume = resume; + } + + @Override + public long getMaxTemplateSizeInBytes() { + return maxTemplateSizeInBytes; + } + + @Override + public void setDownloadError(String error) { + errorString = error; + } + + @Override + public boolean isInited() { + return inited; + } + + public DownloadCommand.ResourceType getResourceType() { + return resourceType; + } + + public Map getDownloadedFilesMap() { + return urlFileMap; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswer.java similarity index 62% rename from core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java rename to core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswer.java index f21211fd3f1a..fd4d9a3ee299 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyAnswer.java @@ -17,31 +17,27 @@ package org.apache.cloudstack.storage.command; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import java.util.List; import com.cloud.agent.api.Answer; -public class PrepareSnapshotZoneCopyAnswer extends Answer { - private SnapshotObjectTO snapshot; +public class QuerySnapshotZoneCopyAnswer extends Answer { + private List files; - public PrepareSnapshotZoneCopyAnswer() { + public QuerySnapshotZoneCopyAnswer() { super(null); } - public PrepareSnapshotZoneCopyAnswer(PrepareSnapshotZoneCopyCommand cmd, SnapshotObjectTO snapshot) { + public QuerySnapshotZoneCopyAnswer(QuerySnapshotZoneCopyCommand cmd, List files) { super(cmd); - setSnapshot(snapshot); + this.files = files; } - public PrepareSnapshotZoneCopyAnswer(PrepareSnapshotZoneCopyCommand cmd, String errMsg) { + public QuerySnapshotZoneCopyAnswer(QuerySnapshotZoneCopyCommand cmd, String errMsg) { super(null, false, errMsg); } - public SnapshotObjectTO getSnapshot() { - return snapshot; - } - - public void setSnapshot(SnapshotObjectTO snapshot) { - this.snapshot = snapshot; + public List getFiles() { + return files; } } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyCommand.java similarity index 84% rename from core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java rename to core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyCommand.java index c9c5c7f3377e..5bca52484ebb 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/PrepareSnapshotZoneCopyCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/QuerySnapshotZoneCopyCommand.java @@ -19,11 +19,15 @@ import org.apache.cloudstack.storage.to.SnapshotObjectTO; -public class PrepareSnapshotZoneCopyCommand extends StorageSubSystemCommand { +/* +Command to get the list of snapshot files for copying a snapshot to a different zone + */ + +public class QuerySnapshotZoneCopyCommand extends StorageSubSystemCommand { private SnapshotObjectTO snapshot; - public PrepareSnapshotZoneCopyCommand(final SnapshotObjectTO snapshot) { + public QuerySnapshotZoneCopyCommand(final SnapshotObjectTO snapshot) { super(); this.snapshot = snapshot; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java index 86f0ab8bcf28..a4a7284abbda 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java @@ -27,11 +27,15 @@ public interface SnapshotDataFactory { SnapshotInfo getSnapshot(DataObject obj, DataStore store); + SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role); + SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role); SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role, boolean retrieveAnySnapshotFromVolume); - List getSnapshots(long volumeId, DataStoreRole store); + List getSnapshotsForVolumeAndStoreRole(long volumeId, DataStoreRole store); + + List getSnapshots(long snapshotId); List listSnapshotOnCache(long snapshotId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java index 802a5a58ef01..d2e085fe90cf 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java @@ -19,6 +19,7 @@ import org.apache.cloudstack.framework.async.AsyncCallFuture; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.storage.Snapshot.Event; public interface SnapshotService { @@ -38,5 +39,7 @@ public interface SnapshotService { void cleanupOnSnapshotBackupFailure(SnapshotInfo snapshot); - AsyncCallFuture copySnapshot(SnapshotInfo snapshot, DataStore dataStore); + AsyncCallFuture copySnapshot(SnapshotInfo snapshot, String copyUrl, DataStore dataStore) throws ResourceUnavailableException; + + AsyncCallFuture queryCopySnapshot(SnapshotInfo snapshot) throws ResourceUnavailableException; } diff --git a/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java b/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java index 6d6264406de5..8474052be201 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VmWorkTakeVolumeSnapshot.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.vm; +import java.util.List; + import com.cloud.storage.Snapshot; public class VmWorkTakeVolumeSnapshot extends VmWork { @@ -29,8 +31,11 @@ public class VmWorkTakeVolumeSnapshot extends VmWork { private Snapshot.LocationType locationType; private boolean asyncBackup; + private List zoneIds; + public VmWorkTakeVolumeSnapshot(long userId, long accountId, long vmId, String handlerName, - Long volumeId, Long policyId, Long snapshotId, boolean quiesceVm, Snapshot.LocationType locationType, boolean asyncBackup) { + Long volumeId, Long policyId, Long snapshotId, boolean quiesceVm, Snapshot.LocationType locationType, + boolean asyncBackup, List zoneIds) { super(userId, accountId, vmId, handlerName); this.volumeId = volumeId; this.policyId = policyId; @@ -38,6 +43,7 @@ public VmWorkTakeVolumeSnapshot(long userId, long accountId, long vmId, String h this.quiesceVm = quiesceVm; this.locationType = locationType; this.asyncBackup = asyncBackup; + this.zoneIds = zoneIds; } public Long getVolumeId() { @@ -61,4 +67,8 @@ public boolean isQuiesceVm() { public boolean isAsyncBackup() { return asyncBackup; } + + public List getZoneIds() { + return zoneIds; + } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 01c7f723ea2f..eef0cdef2fee 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.engine.orchestration; -import com.cloud.capacity.CapacityManager; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -58,6 +57,7 @@ import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.log4j.Logger; +import com.cloud.capacity.CapacityManager; import com.cloud.server.StatsCollector; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; @@ -305,7 +305,7 @@ private void handleSnapshotMigration(Long srcDataStoreId, Date start, Date end, if (!snaps.isEmpty()) { for (SnapshotDataStoreVO snap : snaps) { SnapshotVO snapshotVO = snapshotDao.findById(snap.getSnapshotId()); - SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), DataStoreRole.Image); + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), snap.getDataStoreId(), DataStoreRole.Image); SnapshotInfo parentSnapshot = snapshotInfo.getParent(); if (parentSnapshot == null && policy == MigrationPolicy.COMPLETE) { diff --git a/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java b/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java index ebfad6633ede..8dc9140cbfd4 100644 --- a/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/SnapshotVO.java @@ -16,9 +16,8 @@ // under the License. package com.cloud.storage; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.utils.db.GenericDao; -import com.google.gson.annotations.Expose; +import java.util.Date; +import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; @@ -32,8 +31,9 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; -import java.util.Date; -import java.util.UUID; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.db.GenericDao; +import com.google.gson.annotations.Expose; @Entity @Table(name = "snapshots") @@ -138,6 +138,10 @@ public long getDataCenterId() { return dataCenterId; } + public void setDataCenterId(long dataCenterId) { + this.dataCenterId = dataCenterId; + } + @Override public long getAccountId() { return accountId; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index f70306876147..f317451f1b9e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -44,8 +44,6 @@ public interface SnapshotDataStoreDao extends GenericDao listBySnapshot(long snapshotId, DataStoreRole role); SnapshotDataStoreVO findBySourceSnapshot(long snapshotId, DataStoreRole role); @@ -71,9 +69,7 @@ public interface SnapshotDataStoreDao extends GenericDao findByVolume(long snapshotId, long volumeId, DataStoreRole role); /** * List all snapshots in 'snapshot_store_ref' by volume and data store role. Therefore, it is possible to list all snapshots that are in the primary storage or in the secondary storage. @@ -98,4 +94,6 @@ public interface SnapshotDataStoreDao extends GenericDao listReadyByVolumeId(long volumeId); List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status); + + SnapshotDataStoreVO findOneBySnapshotAndDatastoreRole(long snapshotId, DataStoreRole role); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index a7677147297e..dbb8d76f8efe 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -273,13 +273,6 @@ public SnapshotDataStoreVO findParent(DataStoreRole role, Long storeId, Long vol return null; } - @Override - public SnapshotDataStoreVO findBySnapshot(long snapshotId, DataStoreRole role) { - SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); - sc.setParameters(STATE, State.Ready); - return findOneBy(sc); - } - @Override public List listBySnapshot(long snapshotId, DataStoreRole role) { SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); @@ -303,20 +296,12 @@ public List listAllByVolumeAndDataStore(long volumeId, Data } @Override - public SnapshotDataStoreVO findByVolume(long volumeId, DataStoreRole role) { - SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); - sc.setParameters(VOLUME_ID, volumeId); - sc.setParameters(STORE_ROLE, role); - return findOneBy(sc); - } - - @Override - public SnapshotDataStoreVO findByVolume(long snapshotId, long volumeId, DataStoreRole role) { + public List findByVolume(long snapshotId, long volumeId, DataStoreRole role) { SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(VOLUME_ID, volumeId); sc.setParameters(STORE_ROLE, role); - return findOneBy(sc); + return listBy(sc); } @Override @@ -496,4 +481,11 @@ public List listBySnasphotStoreDownloadStatus(long snapshot sc.setParameters("downloadState", (Object[])status); return search(sc, null); } + + @Override + public SnapshotDataStoreVO findOneBySnapshotAndDatastoreRole(long snapshotId, DataStoreRole role) { + SearchCriteria sc = createSearchCriteriaBySnapshotIdAndStoreRole(snapshotId, role); + sc.setParameters(STATE, State.Ready); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 616569a515a0..214400303230 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -52,7 +52,7 @@ CREATE VIEW `cloud`.`async_job_view` AS async_job.instance_type = 'Template' or async_job.instance_type = 'Iso' THEN - snapshots.uuid + vm_template.uuid WHEN async_job.instance_type = 'VirtualMachine' or async_job.instance_type = 'ConsoleProxy' @@ -91,7 +91,7 @@ CREATE VIEW `cloud`.`async_job_view` AS left join `cloud`.`volumes` ON async_job.instance_id = volumes.id left join - `cloud`.`snapshots` ON async_job.instance_id = snapshots.id + `cloud`.`vm_template` ON async_job.instance_id = vm_template.id left join `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id left join @@ -262,7 +262,9 @@ CREATE VIEW `cloud`.`snapshot_view` AS `resource_tags`.`customer` AS `tag_customer`, CONCAT(`snapshots`.`id`, '_', - IFNULL(`data_center`.`id`, 0)) AS `snapshot_zone_pair` + IFNULL(`snapshot_store_ref`.`store_role`, 'UNKNOWN'), + '_', + IFNULL(`snapshot_store_ref`.`store_id`, 0)) AS `snapshot_store_pair` FROM ((((((((((`snapshots` JOIN `account` ON ((`account`.`id` = `snapshots`.`account_id`))) @@ -285,4 +287,4 @@ CREATE VIEW `cloud`.`snapshot_view` AS OR (`storage_pool`.`data_center_id` = `data_center`.`id`) OR (`snapshot_zone_ref`.`zone_id` = `data_center`.`id`)))) LEFT JOIN `resource_tags` ON ((`resource_tags`.`resource_id` = `snapshots`.`id`) - AND (`resource_tags`.`resource_type` = 'Snapshot'))); \ No newline at end of file + AND (`resource_tags`.`resource_type` = 'Snapshot'))); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java index 1a6a31fafcb7..3557921a8936 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -218,7 +218,7 @@ protected Void migrateDataCallBack(AsyncCallbackDispatcher snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error); - protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storage) { + protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storageToString) { DataTO snapshotTo = snapshot.getTO(); s_logger.debug(String.format("Deleting %s chain of snapshots.", snapshotTo)); @@ -194,7 +190,7 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storage) { SnapshotInfo child = snapshot.getChild(); if (child != null) { - s_logger.debug(String.format("Snapshot [%s] has child [%s], not deleting it on the storage [%s]", snapshotTo, child.getTO(), storage)); + s_logger.debug(String.format("Snapshot [%s] has child [%s], not deleting it on the storage [%s]", snapshotTo, child.getTO(), storageToString)); break; } @@ -233,14 +229,14 @@ protected boolean deleteSnapshotChain(SnapshotInfo snapshot, String storage) { resultIsSet = true; } } catch (Exception e) { - s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storage, e.getMessage()), e); + s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storageToString, e.getMessage()), e); } } snapshot = parent; } } catch (Exception e) { - s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storage, e.getMessage()), e); + s_logger.error(String.format("Failed to delete snapshot [%s] on storage [%s] due to [%s].", snapshotTo, storageToString, e.getMessage()), e); } return result; } @@ -304,11 +300,11 @@ protected void updateSnapshotToDestroyed(SnapshotVO snapshotVo) { } protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo) { - Map snapshotInfos = retrieveSnapshotEntries(snapshotVo.getId()); + List snapshotInfos = retrieveSnapshotEntries(snapshotVo.getId()); boolean result = false; - for (var infoEntry : snapshotInfos.entrySet()) { - if (BooleanUtils.toBooleanDefaultIfNull(deleteSnapshotInfo(infoEntry.getValue(), infoEntry.getKey(), snapshotVo), false)) { + for (var snapshotInfo : snapshotInfos) { + if (BooleanUtils.toBooleanDefaultIfNull(deleteSnapshotInfo(snapshotInfo, snapshotVo), false)) { result = true; } } @@ -320,20 +316,15 @@ protected boolean deleteSnapshotInfos(SnapshotVO snapshotVo) { * Destroys the snapshot entry and file. * @return true if destroy successfully, else false. */ - protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, String storage, SnapshotVO snapshotVo) { - if (snapshotInfo == null) { - s_logger.debug(String.format("Could not find %s entry on %s. Skipping deletion on %s.", snapshotVo, storage, storage)); - return SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER.equals(storage) ? null : true; - } - + protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo) { DataStore dataStore = snapshotInfo.getDataStore(); - String storageToString = String.format("%s {uuid: \"%s\", name: \"%s\"}", storage, dataStore.getUuid(), dataStore.getName()); + String storageToString = String.format("%s {uuid: \"%s\", name: \"%s\"}", dataStore.getRole().name(), dataStore.getUuid(), dataStore.getName()); try { SnapshotObject snapshotObject = castSnapshotInfoToSnapshotObject(snapshotInfo); snapshotObject.processEvent(Snapshot.Event.DestroyRequested); - if (SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER.equals(storage)) { + if (!DataStoreRole.Primary.equals(dataStore.getRole())) { verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObject); @@ -396,13 +387,10 @@ protected SnapshotObject castSnapshotInfoToSnapshotObject(SnapshotInfo snapshotI /** * Retrieves the snapshot infos on primary and secondary storage. * @param snapshotId The snapshot to retrieve the infos. - * @return A map of snapshot infos. + * @return A list of snapshot infos. */ - protected Map retrieveSnapshotEntries(long snapshotId) { - Map snapshotInfos = new LinkedHashMap<>(); - snapshotInfos.put(SECONDARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER, snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image, false)); - snapshotInfos.put(PRIMARY_STORAGE_SNAPSHOT_ENTRY_IDENTIFIER, snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Primary, false)); - return snapshotInfos; + protected List retrieveSnapshotEntries(long snapshotId) { + return snapshotDataFactory.getSnapshots(snapshotId); } @Override diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java index dfe475004f78..b6e82dfa6c89 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/ScaleIOSnapshotStrategy.java @@ -83,7 +83,7 @@ public boolean revertSnapshot(SnapshotInfo snapshotInfo) { } protected boolean isSnapshotStoredOnScaleIOStoragePool(Snapshot snapshot) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return false; } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index d894d7953ffa..786a39637404 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -64,7 +64,7 @@ public SnapshotInfo getSnapshot(DataObject obj, DataStore store) { } @Override - public List getSnapshots(long volumeId, DataStoreRole role) { + public List getSnapshotsForVolumeAndStoreRole(long volumeId, DataStoreRole role) { List allSnapshotsFromVolumeAndDataStore = snapshotStoreDao.listAllByVolumeAndDataStore(volumeId, role); if (CollectionUtils.isEmpty(allSnapshotsFromVolumeAndDataStore)) { return new ArrayList<>(); @@ -83,6 +83,42 @@ public List getSnapshots(long volumeId, DataStoreRole role) { return infos; } + @Override + public List getSnapshots(long snapshotId) { + List allSnapshotsAndDataStore = snapshotStoreDao.findBySnapshotId(snapshotId); + if (CollectionUtils.isEmpty(allSnapshotsAndDataStore)) { + return new ArrayList<>(); + } + List infos = new ArrayList<>(); + for (SnapshotDataStoreVO snapshotDataStoreVO : allSnapshotsAndDataStore) { + DataStore store = storeMgr.getDataStore(snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole()); + SnapshotVO snapshot = snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); + if (snapshot == null){ //snapshot may have been removed; + continue; + } + SnapshotObject info = SnapshotObject.getSnapshotObject(snapshot, store); + + infos.add(info); + } + return infos; + } + + + + @Override + public SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role) { + SnapshotVO snapshot = snapshotDao.findById(snapshotId); + if (snapshot == null) { + return null; + } + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(role, storeId, snapshotId); + if (snapshotStore == null) { + return null; + } + DataStore store = storeMgr.getDataStore(snapshotStore.getDataStoreId(), role); + return SnapshotObject.getSnapshotObject(snapshot, store); + } + @Override public SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role) { return getSnapshot(snapshotId, role, true); @@ -94,13 +130,19 @@ public SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role, boolean ret if (snapshot == null) { return null; } - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshotId, role); + List snapshotStores = snapshotStoreDao.listBySnapshot(snapshotId, role); + SnapshotDataStoreVO snapshotStore = null; + if (CollectionUtils.isNotEmpty(snapshotStores)) { + snapshotStore = snapshotStores.get(0); + } if (snapshotStore == null) { if (!retrieveAnySnapshotFromVolume) { return null; } - - snapshotStore = snapshotStoreDao.findByVolume(snapshotId, snapshot.getVolumeId(), role); + snapshotStores = snapshotStoreDao.findByVolume(snapshotId, snapshot.getVolumeId(), role); + if (CollectionUtils.isNotEmpty(snapshotStores)) { + snapshotStore = snapshotStores.get(0); + } if (snapshotStore == null) { return null; } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 96c898426bb6..4eb67bda1749 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -143,7 +143,7 @@ public List getChildren() { List children = new ArrayList<>(); if (vos != null) { for (SnapshotDataStoreVO vo : vos) { - SnapshotInfo info = snapshotFactory.getSnapshot(vo.getSnapshotId(), DataStoreRole.Image); + SnapshotInfo info = snapshotFactory.getSnapshot(vo.getSnapshotId(), vo.getDataStoreId(), DataStoreRole.Image); if (info != null) { children.add(info); } @@ -165,7 +165,7 @@ public boolean isRevertable() { @Override public long getPhysicalSize() { long physicalSize = 0; - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); if (snapshotStore != null) { physicalSize = snapshotStore.getPhysicalSize(); } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 65dea9c4af9d..08026c27555e 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -49,16 +49,20 @@ import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.configuration.Config; +import com.cloud.dc.DataCenter; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; -import com.cloud.hypervisor.Hypervisor; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.storage.CreateSnapshotPayload; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; @@ -132,6 +136,20 @@ public CopySnapshotContext(AsyncCompletionCallback callback, SnapshotInfo src } + static private class PrepareCopySnapshotContext extends AsyncRpcContext { + final SnapshotInfo snapshot; + final String copyUrlBase; + final AsyncCallFuture future; + + public PrepareCopySnapshotContext(AsyncCompletionCallback callback, SnapshotInfo snapshot, String copyUrlBase, AsyncCallFuture future) { + super(callback); + this.snapshot = snapshot; + this.copyUrlBase = copyUrlBase; + this.future = future; + } + + } + static private class RevertSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; final AsyncCallFuture future; @@ -144,8 +162,7 @@ public RevertSnapshotContext(AsyncCompletionCallback callback, SnapshotInfo s } - private String generateCopyUrl(String ipAddress, String dir, String path) { - String hostname = ipAddress; + private String generateCopyUrlBase(String hostname, String dir) { String scheme = "http"; boolean _sslCopy = false; String sslCfg = _configDao.getValue(Config.SecStorageEncryptCopy.toString()); @@ -159,43 +176,14 @@ private String generateCopyUrl(String ipAddress, String dir, String path) { } if (_sslCopy) { if(_ssvmUrlDomain.startsWith("*")) { - hostname = ipAddress.replace(".", "-"); + hostname = hostname.replace(".", "-"); hostname = hostname + _ssvmUrlDomain.substring(1); } else { hostname = _ssvmUrlDomain; } scheme = "https"; } - return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path; - } - - private String generateVmwareCopyUrl(SnapshotInfo srcSnapshot, EndPoint ep) { -// if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { -// SnapshotObject snapshotObject = (SnapshotObject) srcSnapshot; -// PrepareSnapshotZoneCopyCommand cmd = new PrepareSnapshotZoneCopyCommand((SnapshotObjectTO) snapshotObject.getTO()); -// ep.sendMessageAsync(cmd); -// } - return null; - } - - private String generateCopyUrl(SnapshotInfo srcSnapshot) { - EndPoint ep = epSelector.select(srcSnapshot); - if (ep == null) { - return null; - } - if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { - return generateVmwareCopyUrl(srcSnapshot, ep); - } - if (ep.getPublicAddr() == null) { - s_logger.warn("A running secondary storage vm has a null public ip?"); - return null; - } - String adjustedPath = srcSnapshot.getPath(); - if (Hypervisor.HypervisorType.VMware.equals(srcSnapshot.getHypervisorType())) { - adjustedPath += ".vmdk"; - } - DataStore srcStore = srcSnapshot.getDataStore(); - return generateCopyUrl(ep.getPublicAddr(), ((ImageStoreEntity)srcStore).getMountPoint(), adjustedPath); + return scheme + "://" + hostname + "/copy/SecStorage/" + dir; } protected Void createSnapshotAsyncCallback(AsyncCallbackDispatcher callback, CreateSnapshotContext context) { @@ -317,7 +305,7 @@ private DataStore findSnapshotImageStore(SnapshotInfo snapshot) { // find the image store where the parent snapshot backup is located SnapshotDataStoreVO parentSnapshotOnBackupStore = null; if (parentSnapshot != null) { - parentSnapshotOnBackupStore = _snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); + parentSnapshotOnBackupStore = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(parentSnapshot.getId(), DataStoreRole.Image); } if (parentSnapshotOnBackupStore == null) { return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); @@ -447,6 +435,26 @@ protected Void copySnapshotZoneAsyncCallback(AsyncCallbackDispatcher callback, PrepareCopySnapshotContext context) { + s_logger.info("----------------------executing prepareCopySnapshotZoneAsyncCallback"); + QuerySnapshotZoneCopyAnswer answer = callback.getResult(); + if (answer == null || !answer.getResult()) { + CreateCmdResult result = new CreateCmdResult(null, answer); + result.setResult(answer != null ? answer.getDetails() : "Unsupported answer"); + context.future.complete(result); + return null; + } + List files = answer.getFiles(); + final String copyUrlBase = context.copyUrlBase; + StringBuilder url = new StringBuilder(); + for (String file : files) { + url.append(copyUrlBase).append("/").append(file).append("\n"); + } + CreateCmdResult result = new CreateCmdResult(url.toString().trim(), answer); + context.future.complete(result); + return null; + } + protected Void deleteSnapshotCallback(AsyncCallbackDispatcher callback, DeleteSnapshotContext context) { CommandResult result = callback.getResult(); @@ -700,24 +708,16 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } @Override - public AsyncCallFuture copySnapshot(SnapshotInfo snapshot, DataStore store) { - // generate a URL from source snapshot ssvm to download to destination data store - String url = generateCopyUrl(snapshot); - if (url == null) { - s_logger.warn("Unable to start/resume copy of template " + snapshot.getName() + " to " + store.getName() + - ", no secondary storage vm in running state in source zone"); - throw new CloudRuntimeException("No secondary VM in running state in source template zone "); - } - + public AsyncCallFuture copySnapshot(SnapshotInfo snapshot, String copyUrl, DataStore store) throws ResourceUnavailableException { SnapshotObject snapshotForCopy = (SnapshotObject)_snapshotFactory.getSnapshot(snapshot, store); - snapshotForCopy.setUrl(url); + snapshotForCopy.setUrl(copyUrl); if (s_logger.isDebugEnabled()) { s_logger.debug("Mark template_store_ref entry as Creating"); } AsyncCallFuture future = new AsyncCallFuture(); DataObject snapshotOnStore = store.create(snapshotForCopy); - ((SnapshotObject)snapshotOnStore).setUrl(url); + ((SnapshotObject)snapshotOnStore).setUrl(copyUrl); snapshotOnStore.processEvent(Event.CreateOnlyRequested); if (s_logger.isDebugEnabled()) { @@ -740,4 +740,23 @@ public AsyncCallFuture copySnapshot(SnapshotInfo snapshot, DataS } return future; } + + @Override + public AsyncCallFuture queryCopySnapshot(SnapshotInfo snapshot) throws ResourceUnavailableException { + AsyncCallFuture future = new AsyncCallFuture<>(); + EndPoint ep = epSelector.select(snapshot); + if (ep == null) { + s_logger.error(String.format("Failed to find endpoint for generating copy URL for snapshot %d with store %d", snapshot.getId(), snapshot.getDataStore().getId())); + throw new ResourceUnavailableException("No secondary VM in running state in source snapshot zone", DataCenter.class, snapshot.getDataCenterId()); + } + DataStore store = snapshot.getDataStore(); + String copyUrlBase = generateCopyUrlBase(ep.getPublicAddr(), ((ImageStoreEntity)store).getMountPoint()); + PrepareCopySnapshotContext context = new PrepareCopySnapshotContext<>(null, snapshot, copyUrlBase, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().prepareCopySnapshotZoneAsyncCallback(null, null)).setContext(context); + caller.setContext(context); + QuerySnapshotZoneCopyCommand cmd = new QuerySnapshotZoneCopyCommand((SnapshotObjectTO)(snapshot.getTO())); + ep.sendMessageAsync(cmd, caller); + return future; + } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java index 6401f8a8e1c9..f36607abb306 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java @@ -44,6 +44,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -293,7 +294,7 @@ public boolean revertSnapshot(SnapshotInfo snapshotInfo) { verifySnapshotType(snapshotInfo); - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshotInfo.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshotInfo.getId(), DataStoreRole.Primary); if (snapshotStore != null) { long snapshotStoragePoolId = snapshotStore.getDataStoreId(); @@ -920,14 +921,14 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { return StrategyPriority.CANT_HANDLE; } - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + List snapshotOnImageStores = snapshotStoreDao.listBySnapshot(snapshot.getId(), DataStoreRole.Image); // If the snapshot exists on Secondary Storage, we can't delete it. - if (snapshotStore != null) { + if (CollectionUtils.isNotEmpty(snapshotOnImageStores)) { return StrategyPriority.CANT_HANDLE; } - snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return StrategyPriority.CANT_HANDLE; @@ -953,7 +954,7 @@ public StrategyPriority canHandle(Snapshot snapshot, SnapshotOperation op) { boolean acceptableFormat = isAcceptableRevertFormat(volumeVO); if (acceptableFormat) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); boolean usingBackendSnapshot = usingBackendSnapshotFor(snapshot.getId()); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index a092f8f108eb..7661b2548640 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -18,13 +18,8 @@ package org.apache.cloudstack.storage.snapshot; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import com.cloud.storage.VolumeDetailVO; -import com.cloud.storage.dao.VolumeDetailsDao; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; @@ -41,7 +36,10 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; @RunWith(MockitoJUnitRunner.class) @@ -74,27 +72,22 @@ public class DefaultSnapshotStrategyTest { @Mock SnapshotService snapshotServiceMock; - Map mapStringSnapshotInfoInstance = new LinkedHashMap<>(); + List mockSnapshotInfos = new ArrayList<>(); @Before public void setup() { - mapStringSnapshotInfoInstance.put("secondary storage", snapshotInfo1Mock); - mapStringSnapshotInfoInstance.put("primary storage", snapshotInfo1Mock); + mockSnapshotInfos.add(snapshotInfo1Mock); + mockSnapshotInfos.add(snapshotInfo2Mock); } @Test public void validateRetrieveSnapshotEntries() { Long snapshotId = 1l; - Mockito.doReturn(snapshotInfo1Mock, snapshotInfo2Mock).when(snapshotDataFactoryMock).getSnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.anyBoolean()); - Map result = defaultSnapshotStrategySpy.retrieveSnapshotEntries(snapshotId); - - Mockito.verify(snapshotDataFactoryMock).getSnapshot(snapshotId, DataStoreRole.Image, false); - Mockito.verify(snapshotDataFactoryMock).getSnapshot(snapshotId, DataStoreRole.Primary, false); + Mockito.doReturn(mockSnapshotInfos).when(snapshotDataFactoryMock).getSnapshots(Mockito.anyLong()); + List result = defaultSnapshotStrategySpy.retrieveSnapshotEntries(snapshotId); - Assert.assertTrue(result.containsKey("secondary storage")); - Assert.assertTrue(result.containsKey("primary storage")); - Assert.assertEquals(snapshotInfo1Mock, result.get("secondary storage")); - Assert.assertEquals(snapshotInfo2Mock, result.get("primary storage")); + Assert.assertTrue(result.contains(snapshotInfo1Mock)); + Assert.assertTrue(result.contains(snapshotInfo2Mock)); } @Test @@ -119,36 +112,27 @@ public void validateDestroySnapshotEntriesAndFilesDeletesSuccessfullyReturnsTrue @Test public void validateDeleteSnapshotInfosFailToDeleteReturnsFalse() { - Mockito.doReturn(mapStringSnapshotInfoInstance).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong()); - Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.anyString(), Mockito.any()); + Mockito.doReturn(mockSnapshotInfos).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong()); + Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.any()); Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfos(snapshotVoMock)); } @Test public void validateDeleteSnapshotInfosDeletesSuccessfullyReturnsTrue() { - Mockito.doReturn(mapStringSnapshotInfoInstance).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong()); - Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.anyString(), Mockito.any()); + Mockito.doReturn(mockSnapshotInfos).when(defaultSnapshotStrategySpy).retrieveSnapshotEntries(Mockito.anyLong()); + Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotInfo(Mockito.any(), Mockito.any()); Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInfos(snapshotVoMock)); } - @Test - public void validateDeleteSnapshotInfoSnapshotInfoIsNullOnSecondaryStorageReturnsTrue() { - Assert.assertNull(defaultSnapshotStrategySpy.deleteSnapshotInfo(null, "secondary storage", snapshotVoMock)); - } - - @Test - public void validateDeleteSnapshotInfoSnapshotInfoIsNullOnPrimaryStorageReturnsFalse() { - Assert.assertTrue(defaultSnapshotStrategySpy.deleteSnapshotInfo(null, "primary storage", snapshotVoMock)); - } - @Test public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotOnPrimaryStorage() throws NoTransitionException { Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(true).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "primary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } @@ -158,8 +142,9 @@ public void deleteSnapshotInfoTestReturnFalseIfCannotDeleteTheSnapshotOnPrimaryS Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(false).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "primary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertFalse(result); } @@ -169,8 +154,9 @@ public void deleteSnapshotInfoTestReturnFalseIfDeleteSnapshotOnPrimaryStorageThr Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doThrow(CloudRuntimeException.class).when(snapshotServiceMock).deleteSnapshot(Mockito.any()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Primary); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "primary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertFalse(result); } @@ -181,8 +167,9 @@ public void deleteSnapshotInfoTestReturnTrueIfCanDeleteTheSnapshotChainForSecond Mockito.doNothing().when(defaultSnapshotStrategySpy).verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObjectMock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(true).when(defaultSnapshotStrategySpy).deleteSnapshotChain(Mockito.any(), Mockito.anyString()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Image); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "secondary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } @@ -193,8 +180,9 @@ public void deleteSnapshotInfoTestReturnTrueIfCannotDeleteTheSnapshotChainForSec Mockito.doNothing().when(defaultSnapshotStrategySpy).verifyIfTheSnapshotIsBeingUsedByAnyVolume(snapshotObjectMock); Mockito.doNothing().when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); Mockito.doReturn(false).when(defaultSnapshotStrategySpy).deleteSnapshotChain(Mockito.any(), Mockito.anyString()); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Image); - boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "secondary storage", snapshotVoMock); + boolean result = defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock); Assert.assertTrue(result); } @@ -203,8 +191,9 @@ public void validateDeleteSnapshotInfoSnapshotProcessSnapshotEventThrowsNoTransi Mockito.doReturn(dataStoreMock).when(snapshotInfo1Mock).getDataStore(); Mockito.doReturn(snapshotObjectMock).when(defaultSnapshotStrategySpy).castSnapshotInfoToSnapshotObject(snapshotInfo1Mock); Mockito.doThrow(NoTransitionException.class).when(snapshotObjectMock).processEvent(Mockito.any(Snapshot.Event.class)); + Mockito.when(dataStoreMock.getRole()).thenReturn(DataStoreRole.Image); - Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, "secondary storage", snapshotVoMock)); + Assert.assertFalse(defaultSnapshotStrategySpy.deleteSnapshotInfo(snapshotInfo1Mock, snapshotVoMock)); } @Test diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java index 25de9cd31a74..80c956a2b95e 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImplTest.java @@ -62,7 +62,7 @@ public class SnapshotDataFactoryImplTest { public void getSnapshotsByVolumeAndDataStoreTestNoSnapshotDataStoreVOFound() { Mockito.doReturn(new ArrayList<>()).when(snapshotStoreDaoMock).listAllByVolumeAndDataStore(volumeMockId, DataStoreRole.Primary); - List snapshots = snapshotDataFactoryImpl.getSnapshots(volumeMockId, DataStoreRole.Primary); + List snapshots = snapshotDataFactoryImpl.getSnapshotsForVolumeAndStoreRole(volumeMockId, DataStoreRole.Primary); Assert.assertTrue(snapshots.isEmpty()); } @@ -96,7 +96,7 @@ public void getSnapshotsByVolumeAndDataStoreTest() { Mockito.doReturn(dataStoreMock).when(dataStoreManagerMock).getDataStore(dataStoreId, dataStoreRole); Mockito.doReturn(snapshotVoMock).when(snapshotDaoMock).findById(snapshotId); - List snapshots = snapshotDataFactoryImpl.getSnapshots(volumeMockId, dataStoreRole); + List snapshots = snapshotDataFactoryImpl.getSnapshotsForVolumeAndStoreRole(volumeMockId, dataStoreRole); Assert.assertEquals(1, snapshots.size()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index da97b22946e1..802d1111f268 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -392,7 +392,7 @@ public DataStore findStore(long objId, DataObjectType type, DataStoreRole role) vo = templateDataStoreDao.findByTemplate(objId, role); break; case SNAPSHOT: - vo = snapshotDataStoreDao.findBySnapshot(objId, role); + vo = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(objId, role); break; case VOLUME: vo = volumeDataStoreDao.findByVolume(objId); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 083a6fabecae..b3e8bbfb271a 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -566,12 +566,12 @@ protected void loadPresetVariableValueForSnapshot(UsageVO usageRecord, Value val */ protected long getSnapshotDataStoreId(Long snapshotId) { if (backupSnapshotAfterTakingSnapshot) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshotId, DataStoreRole.Image); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Image); validateIfObjectIsNull(snapshotStore, snapshotId, "data store for snapshot"); return snapshotStore.getDataStoreId(); } - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary); validateIfObjectIsNull(snapshotStore, snapshotId, "data store for snapshot"); return snapshotStore.getDataStoreId(); } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index bfc4bd463f7a..be6a4587311d 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -891,7 +891,7 @@ public void getSnapshotDataStoreIdTestDoNotBackupSnapshotToSecondaryRetrievePrim SnapshotDataStoreVO snapshotDataStoreVoMock = Mockito.mock(SnapshotDataStoreVO.class); Long expected = 1l; - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); Mockito.doReturn(expected).when(snapshotDataStoreVoMock).getDataStoreId(); presetVariableHelperSpy.backupSnapshotAfterTakingSnapshot = false; @@ -901,9 +901,9 @@ public void getSnapshotDataStoreIdTestDoNotBackupSnapshotToSecondaryRetrievePrim Arrays.asList(DataStoreRole.values()).forEach(role -> { if (role == DataStoreRole.Primary) { - Mockito.verify(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.eq(role)); } else { - Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.eq(role)); } }); } @@ -913,7 +913,7 @@ public void getSnapshotDataStoreIdTestBackupSnapshotToSecondaryRetrieveSecondary SnapshotDataStoreVO snapshotDataStoreVoMock = Mockito.mock(SnapshotDataStoreVO.class); Long expected = 2l; - Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); + Mockito.doReturn(snapshotDataStoreVoMock).when(snapshotDataStoreDaoMock).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.any(DataStoreRole.class)); Mockito.doReturn(expected).when(snapshotDataStoreVoMock).getDataStoreId(); presetVariableHelperSpy.backupSnapshotAfterTakingSnapshot = true; @@ -923,9 +923,9 @@ public void getSnapshotDataStoreIdTestBackupSnapshotToSecondaryRetrieveSecondary Arrays.asList(DataStoreRole.values()).forEach(role -> { if (role == DataStoreRole.Image) { - Mockito.verify(snapshotDataStoreDaoMock).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.eq(role)); } else { - Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findBySnapshot(Mockito.anyLong(), Mockito.eq(role)); + Mockito.verify(snapshotDataStoreDaoMock, Mockito.never()).findOneBySnapshotAndDatastoreRole(Mockito.anyLong(), Mockito.eq(role)); } }); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 90bfdd6793c7..cd9919be3c43 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -589,6 +589,7 @@ public Answer executeRequest(Command cmd) { } else if (clz == UnregisterVMCommand.class) { return execute((UnregisterVMCommand) cmd); } else if (cmd instanceof StorageSubSystemCommand) { + s_logger.info("=====================will be handled by storageHandler - " + storageHandler.getClass().getSimpleName()); checkStorageProcessorAndHandlerNfsVersionAttribute((StorageSubSystemCommand) cmd); return storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd); } else if (clz == ScaleVmCommand.class) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java index efe76d84f2ae..3f1a0506038b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java @@ -19,13 +19,21 @@ package com.cloud.storage.resource; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.EnumMap; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyAnswer; -import org.apache.cloudstack.storage.command.PrepareSnapshotZoneCopyCommand; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -205,17 +213,28 @@ protected Answer execute(CopyCommand cmd) { } @Override - protected Answer execute(PrepareSnapshotZoneCopyCommand cmd) { + protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { SnapshotObjectTO snapshot = cmd.getSnapshot(); String parentPath = storageResource.getRootDir(snapshot.getDataStore().getUrl(), _nfsVersion); String path = snapshot.getPath(); int index = path.lastIndexOf(File.separator); - String name = path.substring(index + 1); +// String name = path.substring(index + 1); String snapDir = path.substring(0, index); - int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()), - Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier()); - storageManager.createOva(parentPath + File.separator + snapDir, name, timeout); - snapshot.setPath(snapDir + File.separator + name + ".ova"); - return new PrepareSnapshotZoneCopyAnswer(cmd, snapshot); +// int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()), +// Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier()); +// storageManager.createOva(parentPath + File.separator + snapDir, name, timeout); +// snapshot.setPath(snapDir + File.separator + name + ".ova"); + List files = new ArrayList<>(); + try (Stream stream = Files.list(Paths.get(parentPath + File.separator + snapDir))) { + List fileNames = stream + .filter(file -> !Files.isDirectory(file)) + .map(Path::getFileName) + .map(Path::toString) + .collect(Collectors.toList()); + files.addAll(fileNames); + } catch (IOException ioe) { + s_logger.error("Error preparing file list for snapshot copy", ioe); + } + return new QuerySnapshotZoneCopyAnswer(cmd, files); } } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index cad88dcdd15e..d37a339eb2d9 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -22,13 +22,6 @@ import javax.inject.Inject; -import com.cloud.agent.api.storage.MigrateVolumeCommand; -import com.cloud.agent.api.storage.ResizeVolumeCommand; -import com.cloud.agent.api.to.StorageFilerTO; -import com.cloud.host.HostVO; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -69,13 +62,17 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.host.Host; +import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.server.ManagementServerImpl; import com.cloud.storage.DataStoreRole; @@ -97,7 +94,10 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Preconditions; public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { @@ -915,7 +915,7 @@ public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destDa List snapshots = snapshotDao.listByVolumeId(srcVolumeId); if (CollectionUtils.isNotEmpty(snapshots)) { for (SnapshotVO snapshot : snapshots) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Primary, srcPoolId, snapshot.getId()); if (snapshotStore == null) { continue; } @@ -1086,7 +1086,7 @@ private Answer migrateVolume(DataObject srcData, DataObject destData) { List snapshots = snapshotDao.listByVolumeId(srcData.getId()); if (CollectionUtils.isNotEmpty(snapshots)) { for (SnapshotVO snapshot : snapshots) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findByStoreSnapshot(DataStoreRole.Primary, srcPoolId, snapshot.getId()); if (snapshotStore == null) { continue; } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index 22ad73a118a3..cba35246145c 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -18,6 +18,51 @@ */ package org.apache.cloudstack.storage.datastore.driver; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.RemoteHostEndPoint; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.util.StorPoolHelper; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc; +import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.volume.VolumeObject; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.log4j.Logger; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ResizeVolumeAnswer; import com.cloud.agent.api.storage.StorPoolBackupSnapshotCommand; @@ -61,50 +106,6 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; -import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.framework.async.AsyncCompletionCallback; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.storage.RemoteHostEndPoint; -import org.apache.cloudstack.storage.command.CommandResult; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CreateObjectAnswer; -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.datastore.util.StorPoolHelper; -import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; -import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse; -import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc; -import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.storage.volume.VolumeObject; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.log4j.Logger; - -import javax.inject.Inject; - -import java.util.List; -import java.util.Map; public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { @@ -468,7 +469,7 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal } else if (resp.getError().getName().equals("objectDoesNotExist")) { //check if snapshot is on secondary storage StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snopshot on secondary storage", snapshotName); - SnapshotDataStoreVO snap = snapshotDataStoreDao.findBySnapshot(sinfo.getId(), DataStoreRole.Image); + SnapshotDataStoreVO snap = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(sinfo.getId(), DataStoreRole.Image); // ToDo if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) { resp = StorPoolUtil.volumeCreate(srcData.getUuid(), null, size, null, "no", "snapshot", sinfo.getBaseVolume().getMaxIops(), conn); if (resp.getError() == null) { diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java index 4808ee24e139..9d8bbbc5badb 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java @@ -255,7 +255,7 @@ private boolean deleteSnapshotFromDb(Long snapshotId) { boolean result = deleteSnapshotChain(snapshotOnImage); obj.processEvent(Snapshot.Event.OperationSucceeded); if (result) { - SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); + SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary); if (snapshotOnPrimary != null) { snapshotOnPrimary.setState(State.Destroyed); _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 60f04cf24a4b..fa9d96f6de08 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -696,7 +696,7 @@ public SnapshotResponse createSnapshotResponse(Snapshot snapshot) { } public static DataStoreRole getDataStoreRole(Snapshot snapshot, SnapshotDataStoreDao snapshotStoreDao, DataStoreManager dataStoreMgr) { - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return DataStoreRole.Image; @@ -1914,7 +1914,7 @@ public List createTemplateResponses(ResponseView view, long te // it seems that the volume can actually be removed from the DB at some point if it's deleted // if volume comes back null, use another technique to try to discover the zone if (volume == null) { - SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore != null) { long storagePoolId = snapshotStore.getDataStoreId(); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 891520b3d7ba..6ffc63e90c45 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -4481,8 +4481,17 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps Long zoneId = cmd.getZoneId(); boolean listAll = cmd.listAll(); List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); + Snapshot.LocationType locationType = null; + String locationTypeStr = cmd.getLocationType(); + if (locationTypeStr != null) { + try { + locationType = Snapshot.LocationType.valueOf(locationTypeStr.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(String.format("Invalid %s specified, %s", ApiConstants.LOCATION_TYPE, locationTypeStr)); + } + } - Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotZonePair", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", SortKeyAscending.value(), cmd.getStartIndex(), cmd.getPageSizeVal()); List permittedAccountIds = new ArrayList<>(); Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); @@ -4499,6 +4508,11 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps } SearchBuilder sb = snapshotJoinDao.createSearchBuilder(); + if (cmd.isShowUnique()) { + sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct snapshotId + } else { + sb.select(null, Func.DISTINCT, sb.entity().getSnapshotStorePair()); // select distinct (snapshotId, store_role, store_id) key + } _accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccountIds, listProjectResourcesCriteria); sb.and("statusNEQ", sb.entity().getStatus(), SearchCriteria.Op.NEQ); //exclude those Destroyed snapshot, not showing on UI sb.and("volumeId", sb.entity().getVolumeId(), SearchCriteria.Op.EQ); @@ -4508,6 +4522,7 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps sb.and("snapshotTypeEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.IN); sb.and("snapshotTypeNEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.NIN); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.and("locationType", sb.entity().getStoreRole(), SearchCriteria.Op.EQ); if (tags != null && !tags.isEmpty()) { SearchBuilder tagSearch = _resourceTagDao.createSearchBuilder(); @@ -4554,6 +4569,10 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps sc.setParameters("id", id); } + if (locationType != null) { + sc.setParameters("locationType", locationType.name()); + } + if (keyword != null) { SearchCriteria ssc = snapshotJoinDao.createSearchCriteria(); ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); @@ -4581,7 +4600,27 @@ private Pair, Integer> searchForSnapshotsInternal(ListSnaps sc.setParameters("snapshotTypeNEQ", Snapshot.Type.TEMPLATE.ordinal(), Snapshot.Type.GROUP.ordinal()); } - return snapshotJoinDao.searchIncludingRemovedAndCount(sc, searchFilter); + Pair, Integer> snapshotDataPair; + if (cmd.isShowUnique()) { + snapshotDataPair = snapshotJoinDao.searchAndDistinctCount(sc, searchFilter, new String[]{"snapshot_view.id"}); + } else { + snapshotDataPair = snapshotJoinDao.searchAndDistinctCount(sc, searchFilter, new String[]{"snapshot_view.snapshot_store_pair"}); + } + + Integer count = snapshotDataPair.second(); + if (count == 0) { + // empty result + return snapshotDataPair; + } + List snapshotData = snapshotDataPair.first(); + List snapshots; + if (cmd.isShowUnique()) { + snapshots = snapshotJoinDao.findByDistinctIds(snapshotData.stream().map(SnapshotJoinVO::getId).toArray(Long[]::new)); + } else { + snapshots = snapshotJoinDao.searchBySnapshotStorePair(snapshotData.stream().map(SnapshotJoinVO::getSnapshotStorePair).toArray(String[]::new)); + } + + return new Pair<>(snapshots, count); } @Override diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 39e09d537073..16e76773107b 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -598,7 +598,7 @@ public static List createTemplateResponse(EnumSet createSnapshotResponse(ResponseView view, SnapshotJoinVO... snapshots) { LinkedHashMap vrDataList = new LinkedHashMap<>(); for (SnapshotJoinVO vr : snapshots) { - SnapshotResponse vrData = vrDataList.get(vr.getSnapshotZonePair()); + SnapshotResponse vrData = vrDataList.get(vr.getSnapshotStorePair()); if (vrData == null) { // first time encountering this snapshot vrData = ApiDBUtils.newSnapshotResponse(view, vr); @@ -607,7 +607,7 @@ public static List createSnapshotResponse(ResponseView view, S // update tags vrData = ApiDBUtils.fillSnapshotDetails(view, vrData, vr); } - vrDataList.put(vr.getSnapshotZonePair(), vrData); + vrDataList.put(vr.getSnapshotStorePair(), vrData); } return new ArrayList(vrDataList.values()); } diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java index 3b101e1f257a..651075b1f72b 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java @@ -32,7 +32,10 @@ public interface SnapshotJoinDao extends GenericDao { SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, SnapshotJoinVO snapshotJoinVO); - SnapshotResponse setSnapshotResponse(ResponseObject.ResponseView view, SnapshotResponse snapsData, SnapshotJoinVO snapshot); + SnapshotResponse setSnapshotResponse(ResponseObject.ResponseView view, SnapshotResponse snapshotResponse, SnapshotJoinVO snapshot); Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter); + + List searchBySnapshotStorePair(String... pairs); + List findByDistinctIds(Long... ids); } diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java index 19af823c55cf..806829f25652 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java @@ -17,23 +17,56 @@ package com.cloud.api.query.dao; +import java.util.ArrayList; import java.util.List; +import javax.inject.Inject; + +import org.apache.cloudstack.annotation.AnnotationService; +import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.query.QueryService; import org.apache.log4j.Logger; import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.vo.SnapshotJoinVO; import com.cloud.storage.Snapshot; +import com.cloud.user.AccountService; import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation implements SnapshotJoinDao { public static final Logger s_logger = Logger.getLogger(SnapshotJoinDaoImpl.class); + @Inject + private AccountService accountService; + @Inject + private AnnotationDao annotationDao; + @Inject + private ConfigurationDao configDao; + + private final SearchBuilder snapshotStorePairSearch; + + private final SearchBuilder snapshotIdsSearch; + + SnapshotJoinDaoImpl() { + snapshotStorePairSearch = createSearchBuilder(); + snapshotStorePairSearch.and("snapshotStoreState", snapshotStorePairSearch.entity().getStoreState(), SearchCriteria.Op.IN); + snapshotStorePairSearch.and("snapshotStoreIdIN", snapshotStorePairSearch.entity().getSnapshotStorePair(), SearchCriteria.Op.IN); + snapshotStorePairSearch.done(); + + snapshotIdsSearch = createSearchBuilder(); + snapshotIdsSearch.and("idsIN", snapshotIdsSearch.entity().getId(), SearchCriteria.Op.IN); + snapshotIdsSearch.groupBy(snapshotIdsSearch.entity().getId()); + snapshotIdsSearch.done(); + } + @Override public SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, SnapshotJoinVO snapshot) { SnapshotResponse snapshotResponse = new SnapshotResponse(); @@ -57,24 +90,95 @@ public SnapshotResponse newSnapshotResponse(ResponseObject.ResponseView view, Sn snapshotResponse.setIntervalType(intervalType); snapshotResponse.setState(snapshot.getStatus()); snapshotResponse.setLocationType(snapshot.getLocationType() != null ? snapshot.getLocationType().name() : null); + snapshotResponse.setDatastoreType(snapshot.getStoreRole() != null ? snapshot.getStoreRole().name() : null); // SnapshotInfo snapshotInfo = null; snapshotResponse.setPhysicaSize(snapshot.getStoreSize()); + snapshotResponse.setHasAnnotation(annotationDao.hasAnnotations(snapshot.getUuid(), AnnotationService.EntityType.TEMPLATE.name(), + accountService.isRootAdmin(CallContext.current().getCallingAccount().getId()))); + snapshotResponse.setObjectName("snapshot"); return snapshotResponse; } @Override - public SnapshotResponse setSnapshotResponse(ResponseObject.ResponseView view, SnapshotResponse snapsData, SnapshotJoinVO snapshot) { - return null; + public SnapshotResponse setSnapshotResponse(ResponseObject.ResponseView view, SnapshotResponse snapshotResponse, SnapshotJoinVO snapshot) { + // update tag information + long tag_id = snapshot.getTagId(); + if (tag_id > 0) { + addTagInformation(snapshot, snapshotResponse); + } + + if (snapshotResponse.hasAnnotation() == null) { + snapshotResponse.setHasAnnotation(annotationDao.hasAnnotations(snapshot.getUuid(), AnnotationService.EntityType.TEMPLATE.name(), + accountService.isRootAdmin(CallContext.current().getCallingAccount().getId()))); + } + return snapshotResponse; } @Override public Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter) { List objects = searchIncludingRemoved(sc, filter, null, false); - Integer count = getCountIncludingRemoved(sc); + Integer count = getDistinctCount(sc); return new Pair<>(objects, count); } + + @Override + public List searchBySnapshotStorePair(String... pairs) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + // query details by batches + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", QueryService.SortKeyAscending.value(), null, null); + List uvList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (pairs.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= pairs.length) { + String[] labels = new String[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + labels[k] = pairs[j]; + } + SearchCriteria sc = snapshotStorePairSearch.create(); + sc.setParameters("snapshotStoreIdIN", labels); + List snaps = searchIncludingRemoved(sc, searchFilter, null, false); + if (snaps != null) { + uvList.addAll(snaps); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < pairs.length) { + int batch_size = (pairs.length - curr_index); + String[] labels = new String[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + labels[k] = pairs[j]; + } + SearchCriteria sc = snapshotStorePairSearch.create(); + sc.setParameters("snapshotStoreIdIN", labels); + List vms = searchIncludingRemoved(sc, searchFilter, null, false); + if (vms != null) { + uvList.addAll(vms); + } + } + return uvList; + } + + @Override + public List findByDistinctIds(Long... ids) { + if (ids == null || ids.length == 0) { + return new ArrayList<>(); + } + + Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", QueryService.SortKeyAscending.value(), null, null); + + SearchCriteria sc = snapshotIdsSearch.create(); + sc.setParameters("idsIN", ids); + return searchIncludingRemoved(sc, searchFilter, null, false); + } } diff --git a/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java index 190c586c9771..a4bc23d79903 100644 --- a/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/SnapshotJoinVO.java @@ -28,12 +28,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; import com.cloud.user.Account; import com.cloud.utils.db.GenericDao; -import com.google.gson.annotations.Expose; @Entity @Table(name = "snapshot_view") @@ -66,8 +66,7 @@ public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements Cont @Column(name = GenericDao.REMOVED_COLUMN) Date removed; - @Expose - @Column(name = "location_type", updatable = true, nullable = true) + @Column(name = "location_type") @Enumerated(value = EnumType.STRING) private Snapshot.LocationType locationType; @@ -101,7 +100,7 @@ public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements Cont private String domainPath = null; @Column(name = "project_id") - private long projectId; + private Long projectId; @Column(name = "project_uuid") private String projectUuid; @@ -110,7 +109,7 @@ public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements Cont private String projectName; @Column(name = "data_center_id") - private long dataCenterId; + private Long dataCenterId; @Column(name = "data_center_uuid") private String dataCenterUuid; @@ -138,11 +137,12 @@ public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements Cont private Long storeId; @Column(name = "store_role") - private String storeRole; + @Enumerated(EnumType.STRING) + private DataStoreRole storeRole; @Column(name = "store_state") @Enumerated(EnumType.STRING) - private ObjectInDataStoreStateMachine storeState; + private ObjectInDataStoreStateMachine.State storeState; @Column(name = "download_state") @Enumerated(EnumType.STRING) @@ -160,8 +160,8 @@ public class SnapshotJoinVO extends BaseViewWithTagInformationVO implements Cont @Column(name = "created_on_store") private Date createdOnStore = null; - @Column(name = "snapshot_zone_pair") - private String snapshotZonePair; + @Column(name = "snapshot_store_pair") + private String snapshotStorePair; @Override public String getUuid() { @@ -299,11 +299,11 @@ public Long getStoreId() { return storeId; } - public String getStoreRole() { + public DataStoreRole getStoreRole() { return storeRole; } - public ObjectInDataStoreStateMachine getStoreState() { + public ObjectInDataStoreStateMachine.State getStoreState() { return storeState; } @@ -327,8 +327,8 @@ public Date getCreatedOnStore() { return createdOnStore; } - public String getSnapshotZonePair() { - return snapshotZonePair; + public String getSnapshotStorePair() { + return snapshotStorePair; } @Override diff --git a/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java b/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java index b0bb9eafb2eb..868f785bdc2b 100644 --- a/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java +++ b/server/src/main/java/com/cloud/storage/CreateSnapshotPayload.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage; +import java.util.List; + import com.cloud.user.Account; public class CreateSnapshotPayload { @@ -25,6 +27,7 @@ public class CreateSnapshotPayload { private boolean quiescevm; private Snapshot.LocationType locationType; private boolean asyncBackup; + private List zoneIds; public Long getSnapshotPolicyId() { return snapshotPolicyId; @@ -67,4 +70,12 @@ public void setAsyncBackup(boolean asyncBackup) { public boolean getAsyncBackup() { return this.asyncBackup; } + + public List getZoneIds() { + return zoneIds; + } + + public void setZoneIds(List zoneIds) { + this.zoneIds = zoneIds; + } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 5b1ca9058728..86c3d1228d50 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1317,7 +1317,7 @@ public void cleanupStorage(boolean recurring) { List ssSnapshots = _snapshotStoreDao.listByState(ObjectInDataStoreStateMachine.State.Destroying); for (SnapshotDataStoreVO ssSnapshotVO : ssSnapshots) { try { - _snapshotService.deleteSnapshot(snapshotFactory.getSnapshot(ssSnapshotVO.getSnapshotId(), DataStoreRole.Image)); + _snapshotService.deleteSnapshot(snapshotFactory.getSnapshot(ssSnapshotVO.getSnapshotId(), ssSnapshotVO.getDataStoreId(), DataStoreRole.Image)); } catch (Exception e) { s_logger.debug("Failed to delete snapshot: " + ssSnapshotVO.getId() + " from storage"); } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index a6bccce77aed..1edc0cfb7c1a 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -66,6 +67,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -85,7 +87,9 @@ import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; +import org.apache.cloudstack.resourcedetail.SnapshotPolicyDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; @@ -260,6 +264,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private SnapshotDao _snapshotDao; @Inject + private SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; + @Inject private SnapshotDataStoreDao _snapshotDataStoreDao; @Inject private ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @@ -815,7 +821,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept throw new InvalidParameterValueException("Snapshot id=" + snapshotId + " is not in " + Snapshot.State.BackedUp + " state yet and can't be used for volume creation"); } - SnapshotDataStoreVO snapshotStore = _snapshotDataStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = _snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary); if (snapshotStore != null) { StoragePoolVO storagePoolVO = _storagePoolDao.findById(snapshotStore.getDataStoreId()); if (storagePoolVO.getPoolType() == Storage.StoragePoolType.PowerFlex) { @@ -3393,22 +3399,39 @@ protected Volume liveMigrateVolume(Volume volume, StoragePool destPool) throws S @Override @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "taking snapshot", async = true) - public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) + public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, + Snapshot.LocationType locationType, boolean asyncBackup, Map tags, List zoneIds) throws ResourceAllocationException { - final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup); + final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup, zoneIds); if (snapshot != null && MapUtils.isNotEmpty(tags)) { taggedResourceService.createTags(Collections.singletonList(snapshot.getUuid()), ResourceTag.ResourceObjectType.Snapshot, tags, null); } return snapshot; } - private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, + boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List zoneIds) throws ResourceAllocationException { Account caller = CallContext.current().getCallingAccount(); VolumeInfo volume = volFactory.getVolume(volumeId); if (volume == null) { throw new InvalidParameterValueException("Creating snapshot failed due to volume:" + volumeId + " doesn't exist"); } + if (policyId != null) { + if (CollectionUtils.isNotEmpty(zoneIds)) { + throw new InvalidParameterValueException(String.format("%s can not be specified for snapshots linked with snapshot policy", ApiConstants.ZONE_ID_LIST)); + } + List details = snapshotPolicyDetailsDao.findDetails(policyId, ApiConstants.ZONE_ID); + zoneIds = details.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList()); + } + if (CollectionUtils.isNotEmpty(zoneIds)) { + for (Long destZoneId : zoneIds) { + DataCenterVO dstZone = _dcDao.findById(destZoneId); + if (dstZone == null) { + throw new InvalidParameterValueException("Please specify a valid destination zone."); + } + } + } _accountMgr.checkAccess(caller, null, true, volume); @@ -3437,13 +3460,15 @@ private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapsho VmWorkJobVO placeHolder = null; placeHolder = createPlaceHolderWork(vm.getId()); try { - return orchestrateTakeVolumeSnapshot(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup); + return orchestrateTakeVolumeSnapshot(volumeId, policyId, snapshotId, account, quiescevm, + locationType, asyncBackup, zoneIds); } finally { _workJobDao.expunge(placeHolder.getId()); } } else { - Outcome outcome = takeVolumeSnapshotThroughJobQueue(vm.getId(), volumeId, policyId, snapshotId, account.getId(), quiescevm, locationType, asyncBackup); + Outcome outcome = takeVolumeSnapshotThroughJobQueue(vm.getId(), volumeId, policyId, + snapshotId, account.getId(), quiescevm, locationType, asyncBackup, zoneIds); try { outcome.get(); @@ -3473,12 +3498,16 @@ private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapsho payload.setAccount(account); payload.setQuiescevm(quiescevm); payload.setAsyncBackup(asyncBackup); + if (CollectionUtils.isNotEmpty(zoneIds)) { + payload.setZoneIds(zoneIds); + } volume.addPayload(payload); return volService.takeSnapshot(volume); } } - private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, + boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, List zoneIds) throws ResourceAllocationException { VolumeInfo volume = volFactory.getVolume(volumeId); @@ -3505,6 +3534,9 @@ private Snapshot orchestrateTakeVolumeSnapshot(Long volumeId, Long policyId, Lon payload.setQuiescevm(quiescevm); payload.setLocationType(locationType); payload.setAsyncBackup(asyncBackup); + if (CollectionUtils.isNotEmpty(zoneIds)) { + payload.setZoneIds(zoneIds); + } volume.addPayload(payload); return volService.takeSnapshot(volume); @@ -3529,7 +3561,7 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, } DataCenter zone = _dcDao.findById(volume.getDataCenterId()); if (zone == null) { - throw new InvalidParameterValueException("Can't find zone by id " + volume.getDataCenterId()); + throw new InvalidParameterValueException(String.format("Can't find zone for the volume ID: %s", volume.getUuid())); } if (Grouping.AllocationState.Disabled == zone.getAllocationState() && !_accountMgr.isRootAdmin(caller.getId())) { @@ -3571,9 +3603,18 @@ public Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, } if (CollectionUtils.isNotEmpty(zoneIds)) { + if (policyId != null) { + throw new InvalidParameterValueException(String.format("%s parameter can not be specified with %s parameter", ApiConstants.ZONE_ID_LIST, ApiConstants.POLICY_ID)); + } if (Snapshot.LocationType.PRIMARY.equals(locationType)) { throw new InvalidParameterValueException(String.format("%s cannot be specified with snapshot %s as %s", ApiConstants.ZONE_ID_LIST, ApiConstants.LOCATION_TYPE, Snapshot.LocationType.PRIMARY)); } + if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) { + throw new InvalidParameterValueException("Backing up of snapshot has been disabled. Snapshot can not be taken for multiple zones"); + } + if (DataCenter.Type.Edge.equals(zone.getType())) { + throw new InvalidParameterValueException("Backing up of snapshot is not supported by the zone of the volume. Snapshot can not be taken for multiple zones"); + } for (Long zoneId : zoneIds) { DataCenter dataCenter = _dcDao.findById(zoneId); if (dataCenter == null) { @@ -4679,7 +4720,7 @@ private Outcome migrateVolumeThroughJobQueue(VMInstanceVO vm, VolumeVO v } public Outcome takeVolumeSnapshotThroughJobQueue(final Long vmId, final Long volumeId, final Long policyId, final Long snapshotId, final Long accountId, final boolean quiesceVm, - final Snapshot.LocationType locationType, final boolean asyncBackup) { + final Snapshot.LocationType locationType, final boolean asyncBackup, final List zoneIds) { final CallContext context = CallContext.current(); final User callingUser = context.getCallingUser(); @@ -4701,7 +4742,7 @@ public Outcome takeVolumeSnapshotThroughJobQueue(final Long vmId, fina // save work context info (there are some duplications) VmWorkTakeVolumeSnapshot workInfo = new VmWorkTakeVolumeSnapshot(callingUser.getId(), accountId != null ? accountId : callingAccount.getId(), vm.getId(), - VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, policyId, snapshotId, quiesceVm, locationType, asyncBackup); + VolumeApiServiceImpl.VM_WORK_JOB_HANDLER, volumeId, policyId, snapshotId, quiesceVm, locationType, asyncBackup, zoneIds); workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); @@ -4751,7 +4792,8 @@ private Pair orchestrateMigrateVolume(VmWorkMigrateVolum @ReflectionUse private Pair orchestrateTakeVolumeSnapshot(VmWorkTakeVolumeSnapshot work) throws Exception { Account account = _accountDao.findById(work.getAccountId()); - orchestrateTakeVolumeSnapshot(work.getVolumeId(), work.getPolicyId(), work.getSnapshotId(), account, work.isQuiesceVm(), work.getLocationType(), work.isAsyncBackup()); + orchestrateTakeVolumeSnapshot(work.getVolumeId(), work.getPolicyId(), work.getSnapshotId(), account, + work.isQuiesceVm(), work.getLocationType(), work.isAsyncBackup(), work.getZoneIds()); return new Pair(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(work.getSnapshotId())); } diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java index 7219e0dbb6f0..dd63371b888d 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java @@ -70,8 +70,6 @@ public interface SnapshotManager extends Configurable { */ boolean deleteSnapshotDirsForAccount(long accountId); - String getSecondaryStorageURL(SnapshotVO snapshot); - //void deleteSnapshotsDirForVolume(String secondaryStoragePoolUrl, Long dcId, Long accountId, Long volumeId); boolean canOperateOnVolume(Volume volume); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 4e1ce3c08f56..d451eef3d23b 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -35,6 +36,7 @@ import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.user.snapshot.CopySnapshotCmd; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; @@ -42,6 +44,7 @@ import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -62,6 +65,8 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.resourcedetail.SnapshotPolicyDetailVO; +import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; @@ -70,6 +75,7 @@ import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.log4j.Logger; @@ -97,9 +103,11 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.org.Grouping; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.resource.ResourceManager; import com.cloud.server.ResourceTag.ResourceObjectType; @@ -142,7 +150,6 @@ import com.cloud.utils.DateUtil.IntervalType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; -import com.cloud.utils.StringUtils; import com.cloud.utils.Ternary; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; @@ -180,7 +187,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement @Inject PrimaryDataStoreDao _storagePoolDao; @Inject - SnapshotPolicyDao _snapshotPolicyDao = null; + SnapshotPolicyDao _snapshotPolicyDao; + @Inject + SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; @Inject SnapshotScheduleDao _snapshotScheduleDao; @Inject @@ -640,7 +649,7 @@ public boolean deleteSnapshot(long snapshotId) { DataStoreRole dataStoreRole = snapshotHelper.getDataStoreRole(snapshotCheck); - SnapshotDataStoreVO snapshotStoreRef = _snapshotStoreDao.findBySnapshot(snapshotId, dataStoreRole); + List snapshotStoreRefs = _snapshotStoreDao.listBySnapshot(snapshotId, dataStoreRole); try { boolean result = snapshotStrategy.deleteSnapshot(snapshotId); @@ -658,7 +667,7 @@ public boolean deleteSnapshot(long snapshotId) { } if (snapshotCheck.getState() == Snapshot.State.BackedUp) { - if (snapshotStoreRef != null) { + for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) { _resourceLimitMgr.decrementResourceCount(snapshotCheck.getAccountId(), ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); } } @@ -672,18 +681,6 @@ public boolean deleteSnapshot(long snapshotId) { } } - @Override - public String getSecondaryStorageURL(SnapshotVO snapshot) { - SnapshotDataStoreVO snapshotStore = _snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); - if (snapshotStore != null) { - DataStore store = dataStoreMgr.getDataStore(snapshotStore.getDataStoreId(), DataStoreRole.Image); - if (store != null) { - return store.getUri(); - } - } - throw new CloudRuntimeException("Can not find secondary storage hosting the snapshot"); - } - @Override public Pair, Integer> listSnapshots(ListSnapshotsCmd cmd) { Long volumeId = cmd.getVolumeId(); @@ -853,12 +850,12 @@ public boolean deleteSnapshotDirsForAccount(long accountId) { s_logger.error("Unable to find snapshot strategy to handle snapshot with id '" + snapshot.getId() + "'"); continue; } - SnapshotDataStoreVO snapshotStoreRef = _snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + List snapshotStoreRefs = _snapshotStoreDao.listBySnapshot(snapshot.getId(), DataStoreRole.Image); if (snapshotStrategy.deleteSnapshot(snapshot.getId())) { if (Type.MANUAL == snapshot.getRecurringType()) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.snapshot); - if (snapshotStoreRef != null) { + for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) { _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(snapshotStoreRef.getPhysicalSize())); } } @@ -895,7 +892,8 @@ public SnapshotPolicyVO createPolicy(CreateSnapshotPolicyCmd cmd, Account policy String volumeDescription = volume.getVolumeDescription(); - _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); + final Account caller = CallContext.current().getCallingAccount(); + _accountMgr.checkAccess(caller, null, true, volume); // If display is false we don't actually schedule snapshots. if (volume.getState() != Volume.State.Ready && display) { @@ -974,13 +972,36 @@ public SnapshotPolicyVO createPolicy(CreateSnapshotPolicyCmd cmd, Account policy } } + final List zoneIds = cmd.getZoneIds(); + if (CollectionUtils.isNotEmpty(zoneIds)) { + if (Boolean.FALSE.equals(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value())) { + throw new InvalidParameterValueException("Backing up of snapshot has been disabled. Snapshot can not be taken for multiple zones"); + } + final DataCenterVO zone = dataCenterDao.findById(volume.getDataCenterId()); + if (DataCenter.Type.Edge.equals(zone.getType())) { + throw new InvalidParameterValueException("Backing up of snapshot is not supported by the zone of the volume. Snapshots can not be taken for multiple zones"); + } + for (Long zoneId : zoneIds) { + DataCenter dataCenter = dataCenterDao.findById(zoneId); + if (dataCenter == null) { + throw new InvalidParameterValueException("Unable to find the specified zone"); + } + if (Grouping.AllocationState.Disabled.equals(dataCenter.getAllocationState()) && !_accountMgr.isRootAdmin(caller.getId())) { + throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + dataCenter.getName()); + } + if (DataCenter.Type.Edge.equals(dataCenter.getType())) { + throw new InvalidParameterValueException("Snapshot functionality is not supported on zone %s"); + } + } + } + Map tags = cmd.getTags(); boolean active = true; - return persistSnapshotPolicy(volume, schedule, timezoneId, intvType, maxSnaps, display, active, tags); + return persistSnapshotPolicy(volume, schedule, timezoneId, intvType, maxSnaps, display, active, tags, zoneIds); } - protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, boolean active, Map tags) { + protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, boolean active, Map tags, List zoneIds) { long volumeId = volume.getId(); String volumeDescription = volume.getVolumeDescription(); @@ -997,9 +1018,9 @@ protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedul SnapshotPolicyVO policy = _snapshotPolicyDao.findOneByVolumeInterval(volumeId, intervalType); if (policy == null) { - policy = createSnapshotPolicy(volumeId, schedule, timezone, intervalType, maxSnaps, display); + policy = createSnapshotPolicy(volumeId, schedule, timezone, intervalType, maxSnaps, display, zoneIds); } else { - updateSnapshotPolicy(policy, schedule, timezone, intervalType, maxSnaps, active, display); + updateSnapshotPolicy(policy, schedule, timezone, intervalType, maxSnaps, active, display, zoneIds); } createTagsForSnapshotPolicy(tags, policy); @@ -1011,15 +1032,22 @@ protected SnapshotPolicyVO persistSnapshotPolicy(VolumeVO volume, String schedul } } - protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display) { + protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, List zoneIds) { SnapshotPolicyVO policy = new SnapshotPolicyVO(volumeId, schedule, timezone, intervalType, maxSnaps, display); policy = _snapshotPolicyDao.persist(policy); + if (CollectionUtils.isNotEmpty(zoneIds)) { + List details = new ArrayList<>(); + for (Long zoneId : zoneIds) { + details.add(new SnapshotPolicyDetailVO(policy.getId(), ApiConstants.ZONE_ID, String.valueOf(zoneId))); + } + snapshotPolicyDetailsDao.saveDetails(details); + } _snapSchedMgr.scheduleNextSnapshotJob(policy); s_logger.debug(String.format("Created snapshot policy %s.", new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE).setExcludeFieldNames("id", "uuid", "active"))); return policy; } - protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean active, boolean display) { + protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean active, boolean display, List zoneIds) { String previousPolicy = new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE).setExcludeFieldNames("id", "uuid").toString(); boolean previousDisplay = policy.isDisplay(); policy.setSchedule(schedule); @@ -1029,6 +1057,15 @@ protected void updateSnapshotPolicy(SnapshotPolicyVO policy, String schedule, St policy.setActive(active); policy.setDisplay(display); _snapshotPolicyDao.update(policy.getId(), policy); + if (CollectionUtils.isNotEmpty(zoneIds)) { + List details = snapshotPolicyDetailsDao.listDetails(policy.getId()); + details = details.stream().filter(d -> !ApiConstants.ZONE_ID.equals(d.getName())).collect(Collectors.toList()); + for (Long zoneId : zoneIds) { + details.add(new SnapshotPolicyDetailVO(policy.getId(), ApiConstants.ZONE_ID, String.valueOf(zoneId))); + } + snapshotPolicyDetailsDao.saveDetails(details); + } + _snapSchedMgr.scheduleOrCancelNextSnapshotJobOnDisplayChange(policy, previousDisplay); taggedResourceService.deleteTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, null); s_logger.debug(String.format("Updated snapshot policy %s to %s.", previousPolicy, new ReflectionToStringBuilder(policy, ToStringStyle.JSON_STYLE) @@ -1049,10 +1086,12 @@ public void copySnapshotPoliciesBetweenVolumes(VolumeVO srcVolume, VolumeVO dest s_logger.debug(String.format("Copying snapshot policies %s from volume %s to volume %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(policies, "id", "uuid"), srcVolume.getVolumeDescription(), destVolume.getVolumeDescription())); - policies.forEach(policy -> - persistSnapshotPolicy(destVolume, policy.getSchedule(), policy.getTimezone(), intervalTypes[policy.getInterval()], policy.getMaxSnaps(), - policy.isDisplay(), policy.isActive(), taggedResourceService.getTagsFromResource(ResourceObjectType.SnapshotPolicy, policy.getId())) - ); + for (SnapshotPolicyVO policy : policies) { + List details = snapshotPolicyDetailsDao.findDetails(policy.getId(), ApiConstants.ZONE_ID); + List zoneIds = details.stream().map(d -> Long.valueOf(d.getValue())).collect(Collectors.toList()); + persistSnapshotPolicy(destVolume, policy.getSchedule(), policy.getTimezone(), intervalTypes[policy.getInterval()], policy.getMaxSnaps(), + policy.isDisplay(), policy.isActive(), taggedResourceService.getTagsFromResource(ResourceObjectType.SnapshotPolicy, policy.getId()), zoneIds); + } } protected boolean deletePolicy(Long policyId) { @@ -1296,18 +1335,24 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc try { postCreateSnapshot(volume.getId(), snapshotId, payload.getSnapshotPolicyId()); + snapshotZoneDao.addSnapshotToZone(snapshotId, snapshot.getDataCenterId()); DataStoreRole dataStoreRole = backupSnapToSecondary ? snapshotHelper.getDataStoreRole(snapshot) : DataStoreRole.Primary; - SnapshotDataStoreVO snapshotStoreRef = _snapshotStoreDao.findBySnapshot(snapshotId, dataStoreRole); - if (snapshotStoreRef == null) { + List snapshotStoreRefs = _snapshotStoreDao.listBySnapshot(snapshotId, dataStoreRole); + if (CollectionUtils.isEmpty(snapshotStoreRefs)) { throw new CloudRuntimeException(String.format("Could not find snapshot %s [%s] on [%s]", snapshot.getName(), snapshot.getUuid(), snapshot.getLocationType())); } + SnapshotDataStoreVO snapshotStoreRef = snapshotStoreRefs.get(0); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); // Correct the resource count of snapshot in case of delta snapshots. _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + + if (backupSnapToSecondary) { + copyNewSnapshotToZones(snapshotId, snapshot.getDataCenterId(), payload.getZoneIds()); + } } catch (Exception e) { s_logger.debug("post process snapshot failed", e); } @@ -1493,7 +1538,7 @@ public boolean backedUpSnapshotsExistsForVolume(Volume volume) { @Override public void cleanupSnapshotsByVolume(Long volumeId) { - List infos = snapshotFactory.getSnapshots(volumeId, DataStoreRole.Primary); + List infos = snapshotFactory.getSnapshotsForVolumeAndStoreRole(volumeId, DataStoreRole.Primary); for (SnapshotInfo info : infos) { try { if (info != null) { @@ -1591,85 +1636,6 @@ public void markVolumeSnapshotsAsDestroyed(Volume volume) { } } - @Override - public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException { - Long snapshotId = cmd.getId(); - Long userId = CallContext.current().getCallingUserId(); - Long sourceZoneId = cmd.getSourceZoneId(); - List destZoneIds = cmd.getDestinationZoneIds(); - Account caller = CallContext.current().getCallingAccount(); - - // Verify parameters - SnapshotVO snapshot = _snapshotDao.findById(snapshotId); - if (snapshot == null) { - throw new InvalidParameterValueException("Unable to find snapshot with id"); - } - - // Verify snapshot is backedup and is on Secondary store - if (!Snapshot.State.BackedUp.equals(snapshot.getState())) { - throw new InvalidParameterValueException("Snapshot is not backed up"); - } - if (Snapshot.LocationType.SECONDARY.equals(snapshot.getLocationType())) { - throw new InvalidParameterValueException("Snapshot is not backed up"); - } - if (CollectionUtils.isEmpty(destZoneIds)) { - throw new InvalidParameterValueException("Please specify valid destination zone(s)."); - } - Volume volume = _volsDao.findById(snapshot.getVolumeId()); - if (sourceZoneId == null) { - sourceZoneId = volume.getDataCenterId(); - } - if (destZoneIds.contains(sourceZoneId)) { - throw new InvalidParameterValueException("Please specify different source and destination zones."); - } - DataCenterVO sourceZone = dataCenterDao.findById(sourceZoneId); - if (sourceZone == null) { - throw new InvalidParameterValueException("Please specify a valid source zone."); - } - Map dataCenterVOs = new HashMap<>(); - for (Long destZoneId: destZoneIds) { - DataCenterVO dstZone = dataCenterDao.findById(destZoneId); - if (dstZone == null) { - throw new InvalidParameterValueException("Please specify a valid destination zone."); - } - dataCenterVOs.put(destZoneId, dstZone); - } - - _accountMgr.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, true, snapshot); - - List failedZones = new ArrayList<>(); - - DataStore srcSecStore = getSnapshotZoneImageStore(snapshotId, sourceZoneId); - if (srcSecStore == null) { - throw new InvalidParameterValueException(String.format("There is no snapshot ID: %s ready on image store", snapshot.getUuid())); - } - - for (Long destZoneId : destZoneIds) { - DataStore dstSecStore = getSnapshotZoneImageStore(snapshotId, destZoneId); - if (dstSecStore != null) { - s_logger.debug("There is already snapshot in secondary storage " + dstSecStore.getName() + - " in zone " + destZoneId + " , don't need to copy"); - continue; - } - if (!copy(snapshot, srcSecStore, dataCenterVOs.get(destZoneId))) { - failedZones.add(dataCenterVOs.get(destZoneId).getName()); - } else { - // increase resource count - _resourceLimitMgr.incrementResourceCount(snapshot.getAccountId(), ResourceType.secondary_storage, snapshot.getSize()); - } - } - - if (destZoneIds.size() > failedZones.size()){ - if (!failedZones.isEmpty()) { - s_logger.error("There were failures when copying template to zones: " + - StringUtils.listToCsvTags(failedZones)); - } - return snapshot; - } else { - throw new CloudRuntimeException("Failed to copy template"); - } - } - private boolean checkAndProcessSnapshotAlreadyExistInStore(SnapshotVO snapshot, DataStore dstSecStore) { SnapshotDataStoreVO dstSnapshotStore = _snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, dstSecStore.getId(), snapshot.getId()); if (dstSnapshotStore == null) { @@ -1690,13 +1656,13 @@ private boolean checkAndProcessSnapshotAlreadyExistInStore(SnapshotVO snapshot, } @DB - private boolean copy(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO dstZone) throws StorageUnavailableException, ResourceAllocationException { + private boolean copySnapshotToZone(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO dstZone, String copyUrl) throws StorageUnavailableException, ResourceAllocationException { final long snapshotId = snapshotVO.getId(); - long dstZoneId = dstZone.getId(); + final long dstZoneId = dstZone.getId(); // find all eligible image stores for the destination zone List dstSecStores = dataStoreMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(dstZoneId)); if (CollectionUtils.isEmpty(dstSecStores)) { - throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, dstZone.getId()); + throw new StorageUnavailableException("Destination zone is not ready, no image store associated", DataCenter.class, dstZoneId); } AccountVO account = _accountDao.findById(snapshotVO.getAccountId()); // find the size of the template to be copied @@ -1713,8 +1679,8 @@ private boolean copy(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO return true; } SnapshotInfo snapshotOnSecondary = snapshotFactory.getSnapshot(snapshotId, srcSecStore); - AsyncCallFuture future = snapshotSrv.copySnapshot(snapshotOnSecondary, dstSecStore); try { + AsyncCallFuture future = snapshotSrv.copySnapshot(snapshotOnSecondary, copyUrl, dstSecStore); SnapshotResult result = future.get(); if (result.isFailed()) { s_logger.debug("copy template failed for image store " + dstSecStore.getName() + ":" + result.getResult()); @@ -1728,11 +1694,135 @@ private boolean copy(SnapshotVO snapshotVO, DataStore srcSecStore, DataCenterVO snapshotVO.getSize(), snapshotVO.getClass().getName(), snapshotVO.getUuid()); } return true; - } catch (Exception ex) { + } catch (InterruptedException | ExecutionException | ResourceUnavailableException ex) { s_logger.debug("failed to copy template to image store:" + dstSecStore.getName() + " ,will try next one"); } } return false; + } + + @DB + private List copySnapshotToZones(SnapshotVO snapshotVO, DataStore srcSecStore, List dstZones) throws StorageUnavailableException, ResourceAllocationException { + List failedZones = new ArrayList<>(); + final long snapshotId = snapshotVO.getId(); + SnapshotInfo snapshotOnSecondary = snapshotFactory.getSnapshot(snapshotId, srcSecStore); + String copyUrl = null; + try { + AsyncCallFuture future = snapshotSrv.queryCopySnapshot(snapshotOnSecondary); + CreateCmdResult result = future.get(); + if (!result.isFailed()) { + copyUrl = result.getPath(); + } + } catch (InterruptedException | ExecutionException | ResourceUnavailableException ex) { + throw new CloudRuntimeException("Failed to copy template", ex); + } + if (copyUrl == null) { + throw new CloudRuntimeException("Failed to copy template"); + } + s_logger.debug(String.format("Copying snapshot to destination zones using download URL: %s", copyUrl)); + + for (DataCenterVO destZone : dstZones) { + DataStore dstSecStore = getSnapshotZoneImageStore(snapshotId, destZone.getId()); + if (dstSecStore != null) { + s_logger.debug("There is already snapshot in secondary storage " + dstSecStore.getName() + + " in zone " + destZone.getId() + " , don't need to copy"); + continue; + } + if (copySnapshotToZone(snapshotVO, srcSecStore, destZone, copyUrl)) { + // increase resource count + _resourceLimitMgr.incrementResourceCount(snapshotVO.getAccountId(), ResourceType.secondary_storage, snapshotVO.getSize()); + } else { + failedZones.add(destZone.getName()); + } + } + return failedZones; + } + + private SnapshotVO getCheckedSnapshotForCopy(final long snapshotId, final List destZoneIds, Long sourceZoneId) { + SnapshotVO snapshot = _snapshotDao.findById(snapshotId); + if (snapshot == null) { + throw new InvalidParameterValueException("Unable to find snapshot with id"); + } + // Verify snapshot is BackedUp and is on secondary store + if (!Snapshot.State.BackedUp.equals(snapshot.getState())) { + throw new InvalidParameterValueException("Snapshot is not backed up"); + } + if (Snapshot.LocationType.SECONDARY.equals(snapshot.getLocationType())) { + throw new InvalidParameterValueException("Snapshot is not backed up"); + } + if (CollectionUtils.isEmpty(destZoneIds)) { + throw new InvalidParameterValueException("Please specify valid destination zone(s)."); + } + Volume volume = _volsDao.findById(snapshot.getVolumeId()); + if (sourceZoneId == null) { + sourceZoneId = volume.getDataCenterId(); + } + if (destZoneIds.contains(sourceZoneId)) { + throw new InvalidParameterValueException("Please specify different source and destination zones."); + } + DataCenterVO sourceZone = dataCenterDao.findById(sourceZoneId); + if (sourceZone == null) { + throw new InvalidParameterValueException("Please specify a valid source zone."); + } + // Temporarily set snapshot's zone ID + snapshot.setDataCenterId(sourceZone.getId()); + return snapshot; + } + + @Override + public Snapshot copySnapshot(CopySnapshotCmd cmd) throws StorageUnavailableException, ResourceAllocationException { + final Long snapshotId = cmd.getId(); + Long sourceZoneId = cmd.getSourceZoneId(); + List destZoneIds = cmd.getDestinationZoneIds(); + Account caller = CallContext.current().getCallingAccount(); + SnapshotVO snapshot = getCheckedSnapshotForCopy(snapshotId, destZoneIds, sourceZoneId); + Map dataCenterVOs = new HashMap<>(); + for (Long destZoneId: destZoneIds) { + DataCenterVO dstZone = dataCenterDao.findById(destZoneId); + if (dstZone == null) { + throw new InvalidParameterValueException("Please specify a valid destination zone."); + } + dataCenterVOs.put(destZoneId, dstZone); + } + _accountMgr.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, true, snapshot); + DataStore srcSecStore = getSnapshotZoneImageStore(snapshotId, sourceZoneId); + if (srcSecStore == null) { + throw new InvalidParameterValueException(String.format("There is no snapshot ID: %s ready on image store", snapshot.getUuid())); + } + List failedZones = copySnapshotToZones(snapshot, srcSecStore, new ArrayList<>(dataCenterVOs.values())); + if (destZoneIds.size() > failedZones.size()){ + if (!failedZones.isEmpty()) { + s_logger.error(String.format("There were failures when copying snapshot to zones: %s", + StringUtils.joinWith(", ", failedZones.toArray()))); + } + return snapshot; + } else { + throw new CloudRuntimeException("Failed to copy template"); + } + } + protected void copyNewSnapshotToZones(long snapshotId, long zoneId, List destZoneIds) { + if (CollectionUtils.isEmpty(destZoneIds)) { + return; + } + SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId); + DataStore dataStore = getSnapshotZoneImageStore(snapshotId, zoneId); + if (dataStore == null) { + s_logger.error(String.format("Unable to find an image store for zone ID: %d where snapshot %s is in Ready state", zoneId, snapshotVO)); + } + List dataCenterVOs = new ArrayList<>(); + for (Long destZoneId: destZoneIds) { + DataCenterVO dstZone = dataCenterDao.findById(destZoneId); + dataCenterVOs.add(dstZone); + } + try { + List failedZones = copySnapshotToZones(snapshotVO, dataStore, dataCenterVOs); + if (CollectionUtils.isNotEmpty(failedZones)) { + s_logger.error(String.format("There were failures while copying snapshot %s to zones: %s", + snapshotVO, StringUtils.joinWith(", ", failedZones.toArray()))); + } + } catch (ResourceAllocationException | StorageUnavailableException | CloudRuntimeException e) { + s_logger.error(String.format("Error while copying snapshot %s to zones: %s", snapshotVO, StringUtils.joinWith(",", destZoneIds.toArray()))); + } } } diff --git a/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java b/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java index b00612dca658..d87b4f91f494 100644 --- a/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java +++ b/server/src/main/java/org/apache/cloudstack/snapshot/SnapshotHelper.java @@ -19,17 +19,6 @@ package org.apache.cloudstack.snapshot; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.Storage.StoragePoolType; -import com.cloud.storage.dao.SnapshotDao; - -import com.cloud.utils.exception.CloudRuntimeException; - import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -38,6 +27,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; + import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -57,6 +47,16 @@ import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.log4j.Logger; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.utils.exception.CloudRuntimeException; + public class SnapshotHelper { private final Logger logger = Logger.getLogger(this.getClass()); @@ -168,7 +168,7 @@ public boolean isKvmSnapshotOnlyInPrimaryStorage(Snapshot snapshot, DataStoreRol } public DataStoreRole getDataStoreRole(Snapshot snapshot) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshot.getId(), DataStoreRole.Primary); if (snapshotStore == null) { return DataStoreRole.Image; diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 135a922c082b..d0cce3cbacdb 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -536,7 +536,7 @@ public void testTakeSnapshotF1() throws ResourceAllocationException { when(volumeDataFactoryMock.getVolume(anyLong())).thenReturn(volumeInfoMock); when(volumeInfoMock.getState()).thenReturn(Volume.State.Allocated); lenient().when(volumeInfoMock.getPoolId()).thenReturn(1L); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null); } @Test @@ -549,7 +549,7 @@ public void testTakeSnapshotF2() throws ResourceAllocationException { final TaggedResourceService taggedResourceService = Mockito.mock(TaggedResourceService.class); Mockito.lenient().when(taggedResourceService.createTags(anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(null); ReflectionTestUtils.setField(volumeApiServiceImpl, "taggedResourceService", taggedResourceService); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null, null); } @Test diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java index 8f623d5a4339..49f4657de27b 100755 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java @@ -434,7 +434,7 @@ public void validateCreateSnapshotPolicy(){ Mockito.doReturn(null).when(snapshotSchedulerMock).scheduleNextSnapshotJob(Mockito.any()); SnapshotPolicyVO result = _snapshotMgr.createSnapshotPolicy(TEST_VOLUME_ID, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, null); assertSnapshotPolicyResultAgainstPreBuiltInstance(result); } @@ -449,7 +449,7 @@ public void validateUpdateSnapshotPolicy(){ TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY); _snapshotMgr.updateSnapshotPolicy(snapshotPolicyVo, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, - TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE); + TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null); assertSnapshotPolicyResultAgainstPreBuiltInstance(snapshotPolicyVo); } @@ -484,7 +484,7 @@ public void validatePersistSnapshotPolicyLockIsNotAquiredMustThrowException() { Mockito.doReturn(false).when(globalLockMock).lock(Mockito.anyInt()); _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, - TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, mapStringStringMock); + TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, mapStringStringMock, null); } @Test @@ -498,20 +498,20 @@ public void validatePersistSnapshotPolicyLockAquiredCreateSnapshotPolicy() { Mockito.doReturn(null).when(snapshotPolicyDaoMock).findOneByVolumeInterval(Mockito.anyLong(), Mockito.eq(intervalType)); Mockito.doReturn(snapshotPolicyVoInstance).when(_snapshotMgr).createSnapshotPolicy(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.eq(intervalType), - Mockito.anyInt(), Mockito.anyBoolean()); + Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList()); Mockito.doNothing().when(_snapshotMgr).createTagsForSnapshotPolicy(mapStringStringMock, snapshotPolicyVoMock); SnapshotPolicyVO result = _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, intervalType, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null); assertSnapshotPolicyResultAgainstPreBuiltInstance(result); } VerificationMode timesVerification = Mockito.times(listIntervalTypes.size()); Mockito.verify(_snapshotMgr, timesVerification).createSnapshotPolicy(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.any(DateUtil.IntervalType.class), - Mockito.anyInt(), Mockito.anyBoolean()); + Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList()); Mockito.verify(_snapshotMgr, Mockito.never()).updateSnapshotPolicy(Mockito.any(SnapshotPolicyVO.class), Mockito.anyString(), Mockito.anyString(), - Mockito.any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyList()); Mockito.verify(_snapshotMgr, timesVerification).createTagsForSnapshotPolicy(Mockito.any(), Mockito.any()); } @@ -525,20 +525,20 @@ public void validatePersistSnapshotPolicyLockAquiredUpdateSnapshotPolicy() { for (IntervalType intervalType : listIntervalTypes) { Mockito.doReturn(snapshotPolicyVoInstance).when(snapshotPolicyDaoMock).findOneByVolumeInterval(Mockito.anyLong(), Mockito.eq(intervalType)); Mockito.doNothing().when(_snapshotMgr).updateSnapshotPolicy(Mockito.any(SnapshotPolicyVO.class), Mockito.anyString(), Mockito.anyString(), - Mockito.any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyList()); Mockito.doNothing().when(_snapshotMgr).createTagsForSnapshotPolicy(mapStringStringMock, snapshotPolicyVoMock); SnapshotPolicyVO result = _snapshotMgr.persistSnapshotPolicy(volumeMock, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, intervalType, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null); assertSnapshotPolicyResultAgainstPreBuiltInstance(result); } VerificationMode timesVerification = Mockito.times(listIntervalTypes.size()); Mockito.verify(_snapshotMgr, Mockito.never()).createSnapshotPolicy(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.any(DateUtil.IntervalType.class), - Mockito.anyInt(), Mockito.anyBoolean()); + Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList()); Mockito.verify(_snapshotMgr, timesVerification).updateSnapshotPolicy(Mockito.any(SnapshotPolicyVO.class), Mockito.anyString(), Mockito.anyString(), - Mockito.any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.any(DateUtil.IntervalType.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyList()); Mockito.verify(_snapshotMgr, timesVerification).createTagsForSnapshotPolicy(Mockito.any(), Mockito.any()); } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index e16926e76dcb..8380b44f72ea 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -49,6 +49,8 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.naming.ConfigurationException; @@ -60,6 +62,8 @@ import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.MoveVolumeCommand; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; +import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.command.UploadStatusAnswer; import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; @@ -272,6 +276,7 @@ public static String retrieveNfsVersionFromParams(Map params) { @Override public Answer executeRequest(Command cmd) { + s_logger.info("-----------------------------------------" + cmd.getClass().getSimpleName()); if (cmd instanceof DownloadProgressCommand) { return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd); } else if (cmd instanceof DownloadCommand) { @@ -316,6 +321,8 @@ public Answer executeRequest(Command cmd) { return execute((CreateDatadiskTemplateCommand)cmd); } else if (cmd instanceof MoveVolumeCommand) { return execute((MoveVolumeCommand)cmd); + } else if (cmd instanceof QuerySnapshotZoneCopyCommand) { + return execute((QuerySnapshotZoneCopyCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -3555,4 +3562,28 @@ private TemplateOrVolumePostUploadCommand getTemplateOrVolumePostUploadCmd(Strin return cmd; } + protected Answer execute(QuerySnapshotZoneCopyCommand cmd) { + SnapshotObjectTO snapshot = cmd.getSnapshot(); + String parentPath = getRootDir(snapshot.getDataStore().getUrl(), _nfsVersion); + String path = snapshot.getPath(); + int index = path.lastIndexOf(File.separator); + String snapDir = path.substring(0, index); + List files = new ArrayList<>(); + try (Stream stream = Files.list(Paths.get(parentPath + File.separator + snapDir))) { + List fileNames = stream + .filter(file -> !Files.isDirectory(file)) + .map(Path::getFileName) + .map(Path::toString) + .collect(Collectors.toList()); + for (String file : fileNames) { + file = snapDir + "/" + file; + s_logger.info(String.format("Found snapshot file %s", file)); + files.add(file); + } + } catch (IOException ioe) { + s_logger.error("Error preparing file list for snapshot copy", ioe); + } + return new QuerySnapshotZoneCopyAnswer(cmd, files); + } + } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index dd222a281111..c0490fc0e333 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -64,6 +64,7 @@ import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.SimpleHttpMultiFileDownloader; import com.cloud.storage.template.HttpTemplateDownloader; import com.cloud.storage.template.IsoProcessor; import com.cloud.storage.template.LocalTemplateDownloader; @@ -131,10 +132,8 @@ private static class DownloadJob { private final ResourceType resourceType; private OVFInformationTO ovfInformationTO; - private URI url; - public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, - String installPathPrefix, ResourceType resourceType, URI url) { + String installPathPrefix, ResourceType resourceType) { super(); this.td = td; this.tmpltName = tmpltName; @@ -146,7 +145,6 @@ public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltNam this.templatesize = 0; this.id = id; this.resourceType = resourceType; - this.url = url; } public String getDescription() { @@ -193,17 +191,31 @@ public String getInstallPathPrefix() { return installPathPrefix; } + private void cleanupFileWithDirectory(String path, boolean deleteDir) { + if (StringUtils.isEmpty(path)) { + return; + } + LOGGER.debug(String.format("Cleaning-up temporary download file %s", path)); + File f = new File(path); + File dir = f.getParentFile(); + f.delete(); + if (deleteDir && dir != null) { + LOGGER.debug(String.format("Deleting directory %s, if empty, as part of cleanup", dir.getAbsolutePath())); + dir.delete(); + } + } + public void cleanup() { if (td != null) { - String dnldPath = td.getDownloadLocalPath(); - if (dnldPath != null) { - File f = new File(dnldPath); - File dir = f.getParentFile(); - f.delete(); - if (dir != null) { - dir.delete(); + if (td instanceof SimpleHttpMultiFileDownloader) { + SimpleHttpMultiFileDownloader httpMultiFileDownloader = (SimpleHttpMultiFileDownloader)td; + List files = new ArrayList<>(httpMultiFileDownloader.getDownloadedFilesMap().values()); + for (int i = 0; i < files.size(); ++i) { + cleanupFileWithDirectory(files.get(i), i == files.size() - 1); } + return; } + cleanupFileWithDirectory(td.getDownloadLocalPath(), true); } } @@ -235,14 +247,6 @@ public OVFInformationTO getOvfInformationTO() { public void setOvfInformationTO(OVFInformationTO ovfInformationTO) { this.ovfInformationTO = ovfInformationTO; } - - public URI getUri() { - return url; - } - - public void setUrl(URI url) { - this.url = url; - } } public static final Logger LOGGER = Logger.getLogger(DownloadManagerImpl.class); @@ -380,7 +384,13 @@ private String postRemoteDownload(String jobId) { return result; } - private String getSnapshotInstallNameFromDownloadUrl(URI uri) { + private String getSnapshotInstallNameFromDownloadUrl(String url) { + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException ignored) { + return null; + } String name = uri.getPath(); if (StringUtils.isEmpty(name) || !name.contains("/")) { return null; @@ -394,15 +404,17 @@ private String getSnapshotInstallNameFromDownloadUrl(URI uri) { return name; } - private String postLocalSnapshotDownload(DownloadJob job, String downloadedFile, String resourcePath, String relativeResourcePath) { - String name = getSnapshotInstallNameFromDownloadUrl(job.getUri()); + private String postLocalSnapshotSingleFileDownload(DownloadJob job, HttpTemplateDownloader td) { + String name = getSnapshotInstallNameFromDownloadUrl(td.getDownloadUrl()); + final String downloadedFile = td.getDownloadLocalPath(); + final String resourcePath = job.getInstallPathPrefix(); + final String relativeResourcePath = job.getTmpltPath(); if (StringUtils.isEmpty(name)) { name = UUID.randomUUID().toString(); LOGGER.warn(String.format("Unable to retrieve install filename for snapshot download %s, using a random UUID", downloadedFile)); } Path srcPath = Paths.get(downloadedFile); Path destPath = Paths.get(resourcePath + File.separator + name); - try { LOGGER.debug(String.format("Trying to create missing directories (if any) to move snapshot %s.", destPath)); Files.createDirectories(destPath.getParent()); @@ -420,6 +432,46 @@ private String postLocalSnapshotDownload(DownloadJob job, String downloadedFile, return null; } + private String postLocalSnapshotMultiFileDownload(DownloadJob job, SimpleHttpMultiFileDownloader td) { + Map downloads = td.getDownloadedFilesMap(); + String installDir = null; + try { + for (Map.Entry entry : downloads.entrySet()) { + LOGGER.debug(String.format("Downloaded %s at %s", entry.getKey(), entry.getValue())); + final String url = entry.getKey(); + final String downloadedFile = entry.getValue(); + final String name = url.substring(url.lastIndexOf("/")); + if (StringUtils.isEmpty(installDir)) { + installDir = url.substring(0, url.lastIndexOf("/") - 1); + installDir = installDir.substring(installDir.lastIndexOf("/")); + job.setTmpltPath(job.getTmpltPath() + installDir); + installDir = job.getInstallPathPrefix() + installDir; + Path installPath = Paths.get(installDir); + LOGGER.debug(String.format("Trying to create missing directories (if any) to move snapshot files at %s.", installDir)); + Files.createDirectories(installPath); + } + final String filePath = installDir + name; + Path srcPath = Paths.get(downloadedFile); + Path destPath = Paths.get(filePath); + LOGGER.debug(String.format("Trying to move downloaded snapshot file [%s] to [%s].", srcPath, destPath)); + Files.move(srcPath, destPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + LOGGER.warn(String.format("Something is wrong while processing post snapshot download %s", job.getTmpltPath()), e); + return "Unable process post snapshot download due to " + e.getMessage(); + } + return null; + } + + private String postLocalSnapshotDownload(DownloadJob job, TemplateDownloader td) { + if (td instanceof HttpTemplateDownloader) { + return postLocalSnapshotSingleFileDownload(job, (HttpTemplateDownloader)td); + } else if(td instanceof SimpleHttpMultiFileDownloader) { + return postLocalSnapshotMultiFileDownload(job, (SimpleHttpMultiFileDownloader)td); + } + return null; + } + /** * Post local download activity (install and cleanup). Executed in context of * downloader thread @@ -430,13 +482,12 @@ private String postLocalSnapshotDownload(DownloadJob job, String downloadedFile, private String postLocalDownload(String jobId) { DownloadJob dnld = jobs.get(jobId); TemplateDownloader td = dnld.getTemplateDownloader(); - String resourcePath = dnld.getInstallPathPrefix(); // path with mount directory - String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage ResourceType resourceType = dnld.getResourceType(); - if (ResourceType.SNAPSHOT.equals(resourceType)) { - return postLocalSnapshotDownload(dnld, td.getDownloadLocalPath(), resourcePath, finalResourcePath); + return postLocalSnapshotDownload(dnld, td); } + String resourcePath = dnld.getInstallPathPrefix(); // path with mount directory + String finalResourcePath = dnld.getTmpltPath(); // template download path on secondary storage File originalTemplate = new File(td.getDownloadLocalPath()); if(StringUtils.isBlank(dnld.getChecksum())) { if (LOGGER.isInfoEnabled()) { @@ -627,7 +678,7 @@ public String downloadS3Template(S3TO s3, long id, String url, String name, Imag } else { throw new CloudRuntimeException("Unable to download from URL: " + url); } - DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType, uri); + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); dj.setTmpltPath(installPathPrefix); jobs.put(jobId, dj); threadPool.execute(td); @@ -650,8 +701,9 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm } // TO DO - define constant for volume properties. File file = - ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + - "volume.properties"); + ResourceType.TEMPLATE == resourceType ? + _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : + _storage.getFile(tmpDir + File.separator + "volume.properties"); if (file.exists()) { if(! file.delete()) { LOGGER.warn("Deletion of file '" + file.getAbsolutePath() + "' failed."); @@ -664,37 +716,48 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm } URI uri; + String checkUrl = url; + if (ResourceType.SNAPSHOT.equals(resourceType) && url.contains("\n")) { + checkUrl = url.substring(0, url.indexOf("\n") - 1); + } try { - uri = new URI(url); + uri = new URI(checkUrl); } catch (URISyntaxException e) { throw new CloudRuntimeException("URI is incorrect: " + url); } TemplateDownloader td; - if ((uri != null) && (uri.getScheme() != null)) { - if (uri.getPath().endsWith(".metalink")) { - td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes); - } else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { - td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); - } else if (uri.getScheme().equalsIgnoreCase("file")) { - td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("scp")) { - td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); - } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { - td = null; - // TODO: implement this. - throw new CloudRuntimeException("Scheme is not supported " + url); + if (ResourceType.SNAPSHOT.equals(resourceType) && url.contains("\n") && + ("http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme()))) { + LOGGER.info("~~~~~~using-HttpMultiFileDownloader"); + String[] urls = url.split("\n"); + td = new SimpleHttpMultiFileDownloader(_storage, urls, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, resourceType); + } else { + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getPath().endsWith(".metalink")) { + td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes); + } else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); + } else if (uri.getScheme().equalsIgnoreCase("file")) { + td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("scp")) { + td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); + } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) { + td = null; + // TODO: implement this. + throw new CloudRuntimeException("Scheme is not supported " + url); + } else { + throw new CloudRuntimeException("Scheme is not supported " + url); + } } else { - throw new CloudRuntimeException("Scheme is not supported " + url); + throw new CloudRuntimeException("Unable to download from URL: " + url); } - } else { - throw new CloudRuntimeException("Unable to download from URL: " + url); } // NOTE the difference between installPathPrefix and templatePath // here. instalPathPrefix is the absolute path for template // including mount directory // on ssvm, while templatePath is the final relative path on // secondary storage. - DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType, uri); + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); dj.setTmpltPath(templatePath); jobs.put(jobId, dj); threadPool.execute(td); diff --git a/ui/src/views/storage/FormSchedule.vue b/ui/src/views/storage/FormSchedule.vue index c7cc56b5d26c..478f08e0600e 100644 --- a/ui/src/views/storage/FormSchedule.vue +++ b/ui/src/views/storage/FormSchedule.vue @@ -139,9 +139,9 @@ - +