Allow additional configuration metadata to VM during deployment

api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoPermissionsCmd.java

@@ -45,7 +45,7 @@ protected String getResponseName() {
     @Parameter(name = ApiConstants.ACCOUNTS,
                type = CommandType.LIST,
                collectionType = CommandType.STRING,
-               description = "a comma delimited list of accounts. If specified, \"op\" parameter has to be passed in.")
+               description = "a comma delimited list of accounts within caller's domain. If specified, \"op\" parameter has to be passed in.")
     private List<String> accountNames;
 
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, required = true, description = "the template ID")
@@ -80,7 +80,6 @@ public List<String> getAccountNames() {
         if (accountNames != null && projectIds != null) {
             throw new InvalidParameterValueException("Accounts and projectIds can't be specified together");
         }
-
         return accountNames;
     }
 

api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/RevokeTemplateDirectDownloadCertificateCmd.java

@@ -0,0 +1,98 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.api.command.admin.direct.download;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.direct.download.DirectDownloadManager;
+import org.apache.log4j.Logger;
+
+import javax.inject.Inject;
+
+@APICommand(name = RevokeTemplateDirectDownloadCertificateCmd.APINAME,
+        description = "Revoke a certificate alias from a KVM host",
+        responseObject = SuccessResponse.class,
+        requestHasSensitiveInfo = true,
+        responseHasSensitiveInfo = true,
+        since = "4.13",
+        authorized = {RoleType.Admin})
+public class RevokeTemplateDirectDownloadCertificateCmd extends BaseCmd {
+
+    @Inject
+    DirectDownloadManager directDownloadManager;
+
+    private static final Logger LOG = Logger.getLogger(RevokeTemplateDirectDownloadCertificateCmd.class);
+    public static final String APINAME = "revokeTemplateDirectDownloadCertificate";
+
+    @Parameter(name = ApiConstants.NAME, type = BaseCmd.CommandType.STRING, required = true,
+            description = "alias of the SSL certificate")
+    private String certificateAlias;
+
+    @Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true,
+            description = "hypervisor type")
+    private String hypervisor;
+
+    @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class,
+            description = "zone to revoke certificate", required = true)
+    private Long zoneId;
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
+            description = "(optional) the host ID to revoke certificate")
+    private Long hostId;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        if (!hypervisor.equalsIgnoreCase("kvm")) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Currently supporting KVM hosts only");
+        }
+        SuccessResponse response = new SuccessResponse(getCommandName());
+        try {
+            LOG.debug("Revoking certificate " + certificateAlias + " from " + hypervisor + " hosts");
+            boolean result = directDownloadManager.revokeCertificateAlias(certificateAlias, hypervisor, zoneId, hostId);
+            response.setSuccess(result);
+            setResponseObject(response);
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
+        }
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificateCmd.java

@@ -23,7 +23,9 @@
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.direct.download.DirectDownloadManager;
 import org.apache.log4j.Logger;
@@ -56,6 +58,14 @@ public class UploadTemplateDirectDownloadCertificateCmd extends BaseCmd {
     @Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true, description = "Hypervisor type")
     private String hypervisor;
 
+    @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class,
+            description = "Zone to upload certificate", required = true)
+    private Long zoneId;
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
+            description = "(optional) the host ID to revoke certificate")
+    private Long hostId;
+
     @Override
     public void execute() {
         if (!hypervisor.equalsIgnoreCase("kvm")) {
@@ -64,7 +74,7 @@ public void execute() {
 
         try {
             LOG.debug("Uploading certificate " + name + " to agents for Direct Download");
-            boolean result = directDownloadManager.uploadCertificateToHosts(certificate, name, hypervisor);
+            boolean result = directDownloadManager.uploadCertificateToHosts(certificate, name, hypervisor, zoneId, hostId);
             SuccessResponse response = new SuccessResponse(getCommandName());
             response.setSuccess(result);
             setResponseObject(response);

api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java

@@ -59,6 +59,7 @@ public void execute() {
         response.setKVMSnapshotEnabled((Boolean)capabilities.get("KVMSnapshotEnabled"));
         response.setAllowUserViewDestroyedVM((Boolean)capabilities.get("allowUserViewDestroyedVM"));
         response.setAllowUserExpungeRecoverVM((Boolean)capabilities.get("allowUserExpungeRecoverVM"));
+        response.setAllowUserViewAllDomainAccounts((Boolean)capabilities.get("allowUserViewAllDomainAccounts"));
         if (capabilities.containsKey("apiLimitInterval")) {
             response.setApiLimitInterval((Integer)capabilities.get("apiLimitInterval"));
         }

api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java

@@ -84,6 +84,10 @@ public class CapabilitiesResponse extends BaseResponse {
     @Param(description = "true if the user can recover and expunge virtualmachines, false otherwise", since = "4.6.0")
     private boolean allowUserExpungeRecoverVM;
 
+    @SerializedName("allowuserviewalldomainaccounts")
+    @Param(description = "true if users can see all accounts within the same domain, false otherwise")
+    private boolean allowUserViewAllDomainAccounts;
+
     public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) {
         this.securityGroupsEnabled = securityGroupsEnabled;
     }
@@ -143,4 +147,8 @@ public void setAllowUserViewDestroyedVM(boolean allowUserViewDestroyedVM) {
     public void setAllowUserExpungeRecoverVM(boolean allowUserExpungeRecoverVM) {
         this.allowUserExpungeRecoverVM = allowUserExpungeRecoverVM;
     }
+
+    public void setAllowUserViewAllDomainAccounts(boolean allowUserViewAllDomainAccounts) {
+        this.allowUserViewAllDomainAccounts = allowUserViewAllDomainAccounts;
+    }
 }

api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadCertificate.java

@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.hypervisor.Hypervisor;
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface DirectDownloadCertificate extends InternalIdentity, Identity {
+
+    String getCertificate();
+    String getAlias();
+    Hypervisor.HypervisorType getHypervisorType();
+
+}

api/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManager.java

@@ -19,7 +19,21 @@
 
 import com.cloud.utils.component.PluggableService;
 import org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
 
-public interface DirectDownloadManager extends DirectDownloadService, PluggableService {
+public interface DirectDownloadManager extends DirectDownloadService, PluggableService, Configurable {
 
+    ConfigKey<Long> DirectDownloadCertificateUploadInterval = new ConfigKey<>("Advanced", Long.class,
+            "direct.download.certificate.background.task.interval",
+            "0",
+            "This interval (in hours) controls a background task to sync hosts within enabled zones " +
+                    "missing uploaded certificates for direct download." +
+                    "Only certificates which have not been revoked from hosts are uploaded",
+            false);
+
+    /**
+     * Revoke direct download certificate with alias 'alias' from hosts of hypervisor type 'hypervisor'
+     */
+    boolean revokeCertificateAlias(String certificateAlias, String hypervisor, Long zoneId, Long hostId);
 }

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -103,6 +103,10 @@ public interface QueryService {
                     "network offering, zones), we use the flag to determine if the entities should be sorted ascending (when flag is true) " +
                     "or descending (when flag is false). Within the scope of the config all users see the same result.", true, ConfigKey.Scope.Global);
 
+    public static final ConfigKey<Boolean> AllowUserViewAllDomainAccounts = new ConfigKey<>("Advanced", Boolean.class,
+            "allow.user.view.all.domain.accounts", "false",
+            "Determines whether users can view all user accounts within the same domain", true, ConfigKey.Scope.Domain);
+
     ListResponse<UserResponse> searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException;
 
     ListResponse<EventResponse> searchForEvents(ListEventsCmd cmd);

core/src/main/java/org/apache/cloudstack/agent/directdownload/RevokeDirectDownloadCertificateCommand.java

@@ -0,0 +1,39 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Command;
+
+public class RevokeDirectDownloadCertificateCommand extends Command {
+
+    private String certificateAlias;
+
+    public RevokeDirectDownloadCertificateCommand(final String alias) {
+        this.certificateAlias = alias;
+    }
+
+    public String getCertificateAlias() {
+        return certificateAlias;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -39,9 +39,8 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.agent.api.PrepareForMigrationAnswer;
-import com.cloud.agent.api.to.DpdkTO;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
 import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin;
 import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd;
@@ -95,6 +94,7 @@
 import com.cloud.agent.api.PingRoutingCommand;
 import com.cloud.agent.api.PlugNicAnswer;
 import com.cloud.agent.api.PlugNicCommand;
+import com.cloud.agent.api.PrepareForMigrationAnswer;
 import com.cloud.agent.api.PrepareForMigrationCommand;
 import com.cloud.agent.api.RebootAnswer;
 import com.cloud.agent.api.RebootCommand;
@@ -114,6 +114,7 @@
 import com.cloud.agent.api.UnregisterVMCommand;
 import com.cloud.agent.api.routing.NetworkElementCommand;
 import com.cloud.agent.api.to.DiskTO;
+import com.cloud.agent.api.to.DpdkTO;
 import com.cloud.agent.api.to.GPUDeviceTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
@@ -1119,6 +1120,9 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
 
                     vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
 
+                    // Get VM extraConfig from DB and set to VM TO
+                    addExtraConfig(vmTO);
+
                     work = _workDao.findById(work.getId());
                     if (work == null || work.getStep() != Step.Prepare) {
                         throw new ConcurrentOperationException("Work steps have been changed: " + work);
@@ -1283,6 +1287,16 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
         }
     }
 
+    // Add extra config data to the vmTO as a Map
+    private void addExtraConfig(VirtualMachineTO vmTO) {
+        Map<String, String> details = vmTO.getDetails();
+        for (String key : details.keySet()) {
+            if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
+                vmTO.addExtraConfig(key, details.get(key));
+            }
+        }
+    }
+
     // for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
     private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
         if (hypervisorType != HypervisorType.KVM) {

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -107,4 +107,6 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
      * Side note: this method is currently only used in XenServerGuru; therefore, it was designed to meet XenServer deployment scenarios requirements.
      */
     HostVO findHostInZoneToExecuteCommand(long zoneId, HypervisorType hypervisorType);
+
+    List<HostVO> listAllHostsUpByZoneAndHypervisor(long zoneId, HypervisorType hypervisorType);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.TimeZone;
+import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
@@ -1190,6 +1191,15 @@ public HostVO findHostInZoneToExecuteCommand(long zoneId, HypervisorType hypervi
         }
     }
 
+    @Override
+    public List<HostVO> listAllHostsUpByZoneAndHypervisor(long zoneId, HypervisorType hypervisorType) {
+        return listByDataCenterIdAndHypervisorType(zoneId, hypervisorType)
+                .stream()
+                .filter(x -> x.getStatus().equals(Status.Up) &&
+                        x.getType() == Host.Type.Routing)
+                .collect(Collectors.toList());
+    }
+
     private ResultSet executeSqlGetResultsetForMethodFindHostInZoneToExecuteCommand(HypervisorType hypervisorType, long zoneId, TransactionLegacy tx, String sql) throws SQLException {
         PreparedStatement pstmt = tx.prepareAutoCloseStatement(sql);
         pstmt.setString(1, Objects.toString(hypervisorType));

engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDao.java

@@ -35,4 +35,6 @@ public interface HypervisorCapabilitiesDao extends GenericDao<HypervisorCapabili
     Integer getMaxHostsPerCluster(HypervisorType hypervisorType, String hypervisorVersion);
 
     Boolean isVmSnapshotEnabled(HypervisorType hypervisorType, String hypervisorVersion);
+
+    List<HypervisorType> getHypervisorsWithDefaultEntries();
 }