[WIP] Additional VM config (for review)

.travis.yml

@@ -17,24 +17,28 @@
 sudo: required
 dist: trusty
 group: edge
+
 language: java
 jdk:
-- oraclejdk8
+  - oraclejdk8
 python:
   - "2.7"
+
 cache:
   directories:
-  - $HOME/.m2
+    - $HOME/.m2
   timeout: 500
+
 notifications:
   email: false
+
 env:
   global:
      - PATH=$HOME/.local/bin:$PATH
   matrix:
     # Keep the TESTS sorted by name and grouped by type
     - TESTS="smoke/test_certauthority_root"
-    
+
     - TESTS="smoke/test_accounts
              smoke/test_affinity_groups
              smoke/test_affinity_groups_projects
@@ -43,6 +47,7 @@ env:
              smoke/test_deploy_vm_root_resize
              smoke/test_deploy_vm_with_userdata
              smoke/test_deploy_vms_with_varied_deploymentplanners
+             smoke/test_diagnostics
              smoke/test_disk_offerings
              smoke/test_dynamicroles
              smoke/test_global_settings
@@ -171,3 +176,46 @@ script:
 after_success: ./tools/travis/after_success.sh
 after_failure: ./tools/travis/after_failure.sh
 after_script: ./tools/travis/after_script.sh
+
+# Packaging job definition, will be reused
+.package_job: &package_job
+  before_install: docker pull ${IMAGE}
+  install: true
+  before_script: true
+  script: |
+    docker run \
+     --volume ${TRAVIS_BUILD_DIR}:/mnt/build/cloudstack \
+     --volume $HOME/.m2:/root/.m2 \
+     --rm \
+     ${IMAGE} ${PARAMS}
+  after_script: true
+  after_success: true
+  after_failure: true
+
+jobs:
+  include:
+    - stage: package
+      services: docker
+      env: IMAGE=khos2ow/cloudstack-rpm-builder:centos7 PARAMS="--distribution centos7 --pack oss"
+      <<: *package_job
+
+    - stage: package
+      services: docker
+      env: IMAGE=khos2ow/cloudstack-rpm-builder:centos6 PARAMS="--distribution centos63 --pack oss"
+      <<: *package_job
+
+    - stage: package
+      services: docker
+      env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1804 PARAMS=""
+      <<: *package_job
+
+    - stage: package
+      services: docker
+      env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1604 PARAMS=""
+      <<: *package_job
+
+    - stage: package
+      services: docker
+      env: IMAGE=khos2ow/cloudstack-deb-builder:ubuntu1404 PARAMS=""
+      <<: *package_job
+

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -72,6 +72,18 @@ public class VirtualMachineTO {
 
     Double cpuQuotaPercentage = null;
 
+    boolean enableExtraConfig = false;
+
+    public boolean isExtraConfigEnabled() {
+        return enableExtraConfig;
+    }
+
+    public String getExtraConfig() {
+        return extraConfig;
+    }
+
+    String extraConfig = null;
+
     Map<String, String> guestOsDetails = new HashMap<String, String>();
 
     public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader,
@@ -350,4 +362,12 @@ public Double getCpuQuotaPercentage() {
     public void setCpuQuotaPercentage(Double cpuQuotaPercentage) {
         this.cpuQuotaPercentage = cpuQuotaPercentage;
     }
+
+    public void setEnableExtraConfig(boolean enableExtraConfig) {
+        this.enableExtraConfig = enableExtraConfig;
+    }
+
+    public void setExtraConfig(String extraConfig) {
+        this.extraConfig = extraConfig;
+    }
 }

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -102,4 +102,37 @@ Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account acc
     void updateDisplay(Volume volume, Boolean displayVolume);
 
     Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName) throws ResourceAllocationException;
-}
+
+    /**
+     *  Checks if the target storage supports the disk offering.
+     *  This validation is consistent with the mechanism used to select a storage pool to deploy a volume when a virtual machine is deployed or when a data disk is allocated.
+     *
+     *  The scenarios when this method returns true or false is presented in the following table.
+     *   <table border="1">
+     *      <tr>
+     *          <th>#</th><th>Disk offering tags</th><th>Storage tags</th><th>Does the storage support the disk offering?</th>
+     *      </tr>
+     *      <body>
+     *      <tr>
+     *          <td>1</td><td>A,B</td><td>A</td><td>NO</td>
+     *      </tr>
+     *      <tr>
+     *          <td>2</td><td>A,B,C</td><td>A,B,C,D,X</td><td>YES</td>
+     *      </tr>
+     *      <tr>
+     *          <td>3</td><td>A,B,C</td><td>X,Y,Z</td><td>NO</td>
+     *      </tr>
+     *      <tr>
+     *          <td>4</td><td>null</td><td>A,S,D</td><td>YES</td>
+     *      </tr>
+     *      <tr>
+     *          <td>5</td><td>A</td><td>null</td><td>NO</td>
+     *      </tr>
+     *      <tr>
+     *          <td>6</td><td>null</td><td>null</td><td>YES</td>
+     *      </tr>
+     *      </body>
+     *   </table>
+     */
+    boolean doesTargetStorageSupportDiskOffering(StoragePool destPool, String diskOfferingTags);
+}

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -155,6 +155,7 @@ public class ApiConstants {
     public static final String IDS = "ids";
     public static final String PREVIOUS_ACL_RULE_ID = "previousaclruleid";
     public static final String NEXT_ACL_RULE_ID = "nextaclruleid";
+    public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash";
     public static final String INTERNAL_DNS1 = "internaldns1";
     public static final String INTERNAL_DNS2 = "internaldns2";
     public static final String INTERVAL_TYPE = "intervaltype";
@@ -719,6 +720,13 @@ public class ApiConstants {
     public static final String LAST_ANNOTATED = "lastannotated";
     public static final String LDAP_DOMAIN = "ldapdomain";
 
+    public static final String STDOUT = "stdout";
+    public static final String STDERR = "stderr";
+    public static final String EXITCODE = "exitcode";
+    public static final String TARGET_ID = "targetid";
+
+    public static final String EXTRA_CONFIG = "extraconfig";
+
     public enum HostDetails {
         all, capacity, events, stats, min;
     }

api/src/main/java/org/apache/cloudstack/api/command/admin/diagnostics/RunDiagnosticsCmd.java

@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.diagnostics;
+
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.user.Account;
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.RunDiagnosticsResponse;
+import org.apache.cloudstack.api.response.SystemVmResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.diagnostics.DiagnosticsService;
+import org.apache.cloudstack.diagnostics.DiagnosticsType;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.log4j.Logger;
+
+import javax.inject.Inject;
+import java.util.Collections;
+import java.util.Map;
+
+@APICommand(name = RunDiagnosticsCmd.APINAME, responseObject = RunDiagnosticsResponse.class, entityType = {VirtualMachine.class},
+        responseHasSensitiveInfo = false,
+        requestHasSensitiveInfo = false,
+        description = "Execute network-utility command (ping/arping/tracert) on system VMs remotely",
+        authorized = {RoleType.Admin},
+        since = "4.12.0.0")
+public class RunDiagnosticsCmd extends BaseCmd {
+    private static final Logger LOGGER = Logger.getLogger(RunDiagnosticsCmd.class);
+    public static final String APINAME = "runDiagnostics";
+
+    @Inject
+    private DiagnosticsService diagnosticsService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+    @Parameter(name = ApiConstants.TARGET_ID, type = CommandType.UUID, required = true, entityType = SystemVmResponse.class,
+            validations = {ApiArgValidator.PositiveNumber},
+            description = "The ID of the system VM instance to diagnose")
+    private Long id;
+
+    @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, required = true,
+            validations = {ApiArgValidator.NotNullOrEmpty},
+            description = "The IP/Domain address to test connection to")
+    private String address;
+
+    @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, required = true,
+            validations = {ApiArgValidator.NotNullOrEmpty},
+            description = "The system VM diagnostics type  valid options are: ping, traceroute, arping")
+    private String type;
+
+    @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING,
+            authorized = {RoleType.Admin},
+            description = "Additional command line options that apply for each command")
+    private String optionalArguments;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+    public Long getId() {
+        return id;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public DiagnosticsType getType() {
+        DiagnosticsType diagnosticsType = DiagnosticsType.getCommand(type);
+        if (diagnosticsType == null) {
+            throw new IllegalArgumentException(type + " Is not a valid diagnostics command type. ");
+        }
+        return diagnosticsType;
+    }
+
+    public String getOptionalArguments() {
+        return optionalArguments;
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////////// Implementation //////////////////
+    /////////////////////////////////////////////////////
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        Account account = CallContext.current().getCallingAccount();
+        if (account != null) {
+            return account.getId();
+        }
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException {
+        RunDiagnosticsResponse response = new RunDiagnosticsResponse();
+        try {
+            final Map<String, String> answerMap = diagnosticsService.runDiagnosticsCommand(this);
+            if (CollectionUtils.isNotEmpty(Collections.singleton(answerMap))) {
+                response.setStdout(answerMap.get(ApiConstants.STDOUT));
+                response.setStderr(answerMap.get(ApiConstants.STDERR));
+                response.setExitCode(answerMap.get(ApiConstants.EXITCODE));
+                response.setObjectName("diagnostics");
+                response.setResponseName(getCommandName());
+                this.setResponseObject(response);
+            }
+        } catch (final ServerApiException e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
+        }
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/network/MoveNetworkAclItemCmd.java

@@ -43,6 +43,9 @@ public class MoveNetworkAclItemCmd extends BaseAsyncCustomIdCmd {
     @Parameter(name = ApiConstants.NEXT_ACL_RULE_ID, type = CommandType.STRING, description = "The ID of the rule that is right after the new position where the rule being moved is going to be placed. This value can be 'NULL' if the rule is being moved to the last position of the network ACL list.")
     private String nextAclRuleUuid;
 
+    @Parameter(name = ApiConstants.MOVE_ACL_CONSISTENCY_HASH, type = CommandType.STRING, description = "Md5 hash used to check the consistency of the ACL rule list before applying the ACL rule move. This check is useful to manage concurrency problems that may happen when multiple users are editing the same ACL rule listing. The parameter is not required. Therefore, if the user does not send it, he/she is assuming the risk of moving ACL rules without checking the consistency of the access control list before executing the move. We use MD5 hash function on a String that is composed of all UUIDs of the ACL rules in concatenated in their respective order (order defined via 'number' field).")
+    private String aclConsistencyHash;
+
     @Override
     public void execute() {
         CallContext.current().setEventDetails(getEventDescription());
@@ -93,4 +96,8 @@ public long getEntityOwnerId() {
         Account caller = CallContext.current().getCallingAccount();
         return caller.getAccountId();
     }
-}
+
+    public String getAclConsistencyHash() {
+        return aclConsistencyHash;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -200,6 +200,9 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
             " an optional parameter used to create additional data disks from datadisk templates; can't be specified with diskOfferingId parameter")
     private Map dataDiskTemplateToDiskOfferingList;
 
+    @Parameter(name = ApiConstants.EXTRA_CONFIG, type = CommandType.STRING, since = "4.12", description = "an optional URL encoded string that can be passed to the virtual machine upon successful deployment", authorized = { RoleType.Admin }, length = 5120)
+    private String extraConfig;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -482,6 +485,10 @@ public Map<Long, DiskOffering> getDataDiskTemplateToDiskOfferingMap() {
         return dataDiskTemplateToDiskOfferingMap;
     }
 
+    public String getExtraConfig() {
+        return extraConfig;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/ExtractResponse.java

@@ -66,9 +66,6 @@ public class ExtractResponse extends BaseResponse {
     @Param(description = "type of the storage")
     private String storageType;
 
-    @SerializedName("storage")
-    private String storage;
-
     @SerializedName(ApiConstants.ZONE_ID)
     @Param(description = "zone ID the object was extracted from")
     private String zoneId;
@@ -176,14 +173,6 @@ public void setStorageType(String storageType) {
         this.storageType = storageType;
     }
 
-    public String getStorage() {
-        return storage;
-    }
-
-    public void setStorage(String storage) {
-        this.storage = storage;
-    }
-
     public String getZoneId() {
         return zoneId;
     }