WIP: Fix primary storage usage for vm snaphot

api/src/com/cloud/event/EventTypes.java

@@ -233,6 +233,8 @@ public class EventTypes {
 
     // Snapshots
     public static final String EVENT_SNAPSHOT_CREATE = "SNAPSHOT.CREATE";
+    public static final String EVENT_SNAPSHOT_ON_PRIMARY = "SNAPSHOT.ON_PRIMARY";
+    public static final String EVENT_SNAPSHOT_OFF_PRIMARY = "SNAPSHOT.OFF_PRIMARY";
     public static final String EVENT_SNAPSHOT_DELETE = "SNAPSHOT.DELETE";
     public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT";
     public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE";
@@ -441,6 +443,8 @@ public class EventTypes {
     // vm snapshot events
     public static final String EVENT_VM_SNAPSHOT_CREATE = "VMSNAPSHOT.CREATE";
     public static final String EVENT_VM_SNAPSHOT_DELETE = "VMSNAPSHOT.DELETE";
+    public static final String EVENT_VM_SNAPSHOT_ON_PRIMARY = "VMSNAPSHOT.ON_PRIMARY";
+    public static final String EVENT_VM_SNAPSHOT_OFF_PRIMARY = "VMSNAPSHOT.OFF_PRIMARY";
     public static final String EVENT_VM_SNAPSHOT_REVERT = "VMSNAPSHOT.REVERTTO";
 
     // external network device events
@@ -679,6 +683,8 @@ public class EventTypes {
         // Snapshots
         entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class);
+        entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class);
+        entityEventDetails.put(EVENT_SNAPSHOT_OFF_PRIMARY, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_POLICY_CREATE, SnapshotPolicy.class);
         entityEventDetails.put(EVENT_SNAPSHOT_POLICY_UPDATE, SnapshotPolicy.class);
         entityEventDetails.put(EVENT_SNAPSHOT_POLICY_DELETE, SnapshotPolicy.class);

engine/schema/src/com/cloud/usage/UsageSnapshotOnPrimaryVO.java

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.usage;
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+import org.apache.cloudstack.api.InternalIdentity;
+
+@Entity
+@Table(name = "usage_snapshot_on_primary")
+public class UsageSnapshotOnPrimaryVO implements InternalIdentity {
+
+    @Column(name = "id")
+    // volumeId
+    private long id;
+
+    @Column(name = "zone_id")
+    private long zoneId;
+
+    @Column(name = "account_id")
+    private long accountId;
+
+    @Column(name = "domain_id")
+    private long domainId;
+
+    @Column(name = "vm_id")
+    private long vmId;
+
+    @Column(name = "type")
+    private int snapshotType;
+
+    @Column(name = "size")
+    private long size;
+
+    @Column(name = "created")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date created = null;
+
+    @Column(name = "processed")
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date processed;
+
+    protected UsageSnapshotOnPrimaryVO() {
+    }
+
+    public UsageSnapshotOnPrimaryVO(long id, long zoneId, long accountId, long domainId, long vmId, int type, long size, Date created, Date processed) {
+        this.zoneId = zoneId;
+        this.accountId = accountId;
+        this.domainId = domainId;
+        this.snapshotType = type;
+        this.id = id;
+        this.size = size;
+        this.created = created;
+        this.vmId = vmId;
+        this.processed = processed;
+    }
+
+    public long getZoneId() {
+        return zoneId;
+    }
+
+    public long getAccountId() {
+        return accountId;
+    }
+
+    public long getDomainId() {
+        return domainId;
+    }
+
+    public int getSnapshotType() {
+        return snapshotType;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    public Date getProcessed() {
+        return processed;
+    }
+
+    public void setProcessed(Date processed) {
+        this.processed = processed;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    @Override
+    public long getId() {
+        return this.id;
+    }
+
+}
+

engine/schema/src/com/cloud/usage/dao/UsageSnapshotOnPrimaryDao.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.usage.dao;
+
+import java.util.Date;
+import java.util.List;
+
+import com.cloud.usage.UsageSnapshotOnPrimaryVO;import com.cloud.utils.db.GenericDao;
+
+public interface UsageSnapshotOnPrimaryDao extends GenericDao<UsageSnapshotOnPrimaryVO, Long> {
+    public void update(UsageSnapshotOnPrimaryVO usage);
+
+    public List<UsageSnapshotOnPrimaryVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate);
+
+    UsageSnapshotOnPrimaryVO getPreviousUsageRecord(UsageSnapshotOnPrimaryVO rec);
+}
+

engine/schema/src/com/cloud/usage/dao/UsageSnapshotOnPrimaryDaoImpl.java

@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.usage.dao;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
+import com.cloud.usage.UsageSnapshotOnPrimaryVO;
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.TransactionLegacy;
+
+@Component
+public class UsageSnapshotOnPrimaryDaoImpl extends GenericDaoBase<UsageSnapshotOnPrimaryVO, Long> implements UsageSnapshotOnPrimaryDao {
+    public static final Logger s_logger = Logger.getLogger(UsageSnapshotOnPrimaryDaoImpl.class.getName());
+    protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, zone_id, account_id, domain_id, vm_id, type, size, created, processed "
+        + " FROM usage_snapshot_on_primary" + " WHERE account_id = ? " + " AND ( (created BETWEEN ? AND ?) OR "
+        + "      (created < ? AND processed is NULL) ) ORDER BY created asc";
+    protected static final String UPDATE_DELETED = "UPDATE usage_snapshot_on_primary SET processed = ? WHERE account_id = ? AND id = ? and vm_id = ?  and created = ?";
+
+    protected static final String PREVIOUS_QUERY = "SELECT id, zone_id, account_id, domain_id, vm_id, type, size, created, processed "
+        + "FROM usage_snapshot_on_primary " + "WHERE account_id = ? AND id = ? AND vm_id = ? AND created < ? AND processed IS NULL " + "ORDER BY created desc limit 1";
+
+    @Override
+    public void update(UsageSnapshotOnPrimaryVO usage) {
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
+        PreparedStatement pstmt = null;
+        try {
+            txn.start();
+            pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED);
+            pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getProcessed()));
+            pstmt.setLong(2, usage.getAccountId());
+            pstmt.setLong(3, usage.getId());
+            pstmt.setLong(4, usage.getVmId());
+            pstmt.setString(5, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getCreated()));
+            pstmt.executeUpdate();
+            txn.commit();
+        } catch (Exception e) {
+            txn.rollback();
+            s_logger.warn("Error updating UsageSnapshotOnPrimaryVO", e);
+        } finally {
+            txn.close();
+        }
+    }
+
+    @Override
+    public List<UsageSnapshotOnPrimaryVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate) {
+        List<UsageSnapshotOnPrimaryVO> usageRecords = new ArrayList<UsageSnapshotOnPrimaryVO>();
+
+        String sql = GET_USAGE_RECORDS_BY_ACCOUNT;
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
+        PreparedStatement pstmt = null;
+
+        try {
+            int i = 1;
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            pstmt.setLong(i++, accountId);
+            pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate));
+            pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate));
+            pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate));
+
+            ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                //id, zone_id, account_id, domain_iVMSnapshotVOd, vm_id, disk_offering_id, size, created, processed
+                Long vId = Long.valueOf(rs.getLong(1));
+                Long zoneId = Long.valueOf(rs.getLong(2));
+                Long acctId = Long.valueOf(rs.getLong(3));
+                Long dId = Long.valueOf(rs.getLong(4));
+                Long vmId = Long.valueOf(rs.getLong(5));
+                Integer type = Integer.valueOf(rs.getInt(6));
+                Long size = Long.valueOf(rs.getLong(7));
+                Date createdDate = null;
+                Date processDate = null;
+                String createdTS = rs.getString(8);
+                String processed = rs.getString(9);
+
+                if (createdTS != null) {
+                    createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS);
+                }
+                if (processed != null) {
+                    processDate = DateUtil.parseDateString(s_gmtTimeZone, processed);
+                }
+                usageRecords.add(new UsageSnapshotOnPrimaryVO(vId, zoneId, acctId, dId, vmId, type, size, createdDate, processDate));
+            }
+        } catch (Exception e) {
+            txn.rollback();
+            s_logger.warn("Error getting usage records", e);
+        } finally {
+            txn.close();
+        }
+
+        return usageRecords;
+    }
+
+    @Override
+    public UsageSnapshotOnPrimaryVO getPreviousUsageRecord(UsageSnapshotOnPrimaryVO rec) {
+        List<UsageSnapshotOnPrimaryVO> usageRecords = new ArrayList<UsageSnapshotOnPrimaryVO>();
+
+        String sql = PREVIOUS_QUERY;
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
+        PreparedStatement pstmt = null;
+        try {
+            int i = 1;
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            pstmt.setLong(i++, rec.getAccountId());
+            pstmt.setLong(i++, rec.getId());
+            pstmt.setLong(i++, rec.getVmId());
+            pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), rec.getCreated()));
+
+            ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                //id, zone_id, account_id, domain_iVMSnapshotVOd, vm_id, disk_offering_id, size, created, processed
+                Long vId = Long.valueOf(rs.getLong(1));
+                Long zoneId = Long.valueOf(rs.getLong(2));
+                Long acctId = Long.valueOf(rs.getLong(3));
+                Long dId = Long.valueOf(rs.getLong(4));
+                Long vmId = Long.valueOf(rs.getLong(5));
+                Integer type = Integer.valueOf(rs.getInt(6));
+                Long size = Long.valueOf(rs.getLong(7));
+                Date createdDate = null;
+                Date processDate = null;
+                String createdTS = rs.getString(8);
+                String processed = rs.getString(9);
+
+                if (createdTS != null) {
+                    createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS);
+                }
+                if (processed != null) {
+                    processDate = DateUtil.parseDateString(s_gmtTimeZone, processed);
+                }
+                usageRecords.add(new UsageSnapshotOnPrimaryVO(vId, zoneId, acctId, dId, vmId, type, size, createdDate, processDate));
+            }
+        } catch (Exception e) {
+            txn.rollback();
+            s_logger.warn("Error getting usage records", e);
+        } finally {
+            txn.close();
+        }
+
+        if (usageRecords.size() > 0)
+            return usageRecords.get(0);
+        return null;
+    }
+}

engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -548,6 +548,7 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
                 addFullCloneFlagOnVMwareDest(destData.getTO());
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
                 cmd.setOptions(options);
+                s_logger.error("IR24 copySnapshot BACKUPSNAPSHOT " + srcData.getId() + " dest=" + destData.getId());
                 EndPoint ep = selector.select(srcData, destData, StorageAction.BACKUPSNAPSHOT);
                 if (ep == null) {
                     String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";

engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java

@@ -49,6 +49,8 @@
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
 
+import com.cloud.event.EventTypes;
+import com.cloud.event.UsageEventUtils;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotVO;
@@ -195,7 +197,7 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) {
             AsyncCallbackDispatcher<SnapshotServiceImpl, CreateCmdResult> caller = AsyncCallbackDispatcher.create(this);
             caller.setCallback(caller.getTarget().createSnapshotAsyncCallback(null, null)).setContext(context);
             PrimaryDataStoreDriver primaryStore = (PrimaryDataStoreDriver)snapshotOnPrimary.getDataStore().getDriver();
-            primaryStore.takeSnapshot(snapshot, caller);
+            primaryStore.takeSnapshot(snapshot, caller); // IR24 take snapshot on primary
         } catch (Exception e) {
             s_logger.debug("Failed to take snapshot: " + snapshot.getId(), e);
             try {
@@ -211,6 +213,11 @@ public SnapshotResult takeSnapshot(SnapshotInfo snap) {
 
         try {
             result = future.get();
+            s_logger.debug("IR24 creating snapshot on primary " + snap.getName() + " snapshotOnPrimary=" + snapshotOnPrimary.getSize() +
+                    " snapshotOnPrimary.getPhysicalSize()=" + snapshotOnPrimary.getPhysicalSize());
+            // IR24 generate the off primary event
+            UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_ON_PRIMARY, snap.getAccountId(), snap.getDataCenterId(), snap.getId(),
+                    snap.getName(), null, null, snapshotOnPrimary.getSize(), snapshotOnPrimary.getSize(), snap.getClass().getName(), snap.getUuid());
             return result;
         } catch (InterruptedException e) {
             s_logger.debug("Failed to create snapshot", e);