DOCKER-863: Create shared volume when mounting nonexistent volume

DOCKER-881: docker start should provision shared volume only if required
VOLAPI-26: wf-runner crashes with: Uncaught AssertionError: "string" == "object"
VOLAPI-9 Implement asynchronous NFS shared volume creation
VOLAPI-6: Deleting a tritonnfs volume used by an active Docker container should fail
VOLAPI-27 Divergence in `docker create -v` behavior; volume is not created until `docker start`
VOLAPI-54 Custom errors' rest codes should follow engineering guide's guidelines
VOLAPI-55 After enabling volapi, a sdcadm update without volapi fails
DOCKER-1034 support docker volume ls -f dangling=true
VOLAPI-65 rename required_nfs_volumes VM objects' property to volumes
TOOLS-1830 replace sdcadm experimental nfs-volumes with cloudapi-nfs-volumes/docker-nfs-volumes subcommands
ZAPI-796 NFS volumes provisioning tasks chain broken by switch from node-uuid to uuid module in sdc-workflow
DOCKER-1088 sdc-docker should not modify VMs' internal docker:nfsvolumes metadata
PUBAPI-1420 Add ability to mount NFS volumes with CreateMachine endpoint
ZAPI-794 remove_volume_references task in destroy workflow should not error

lib/apis/moray.js

@@ -942,6 +942,8 @@ Moray.prototype.markAsDestroyed = function markAsDestroyed(vm, callback) {
 
     var self = this;
 
+    self._log.debug({vm: vm}, 'Marking VM as destroyed');
+
     var oldVm = jsprim.deepCopy(vm);
 
     if (!self.bucketsSetup()) {

lib/common/validation.js

@@ -22,6 +22,7 @@ var net = require('net');
 var restify = require('restify');
 var strsplit = require('strsplit');
 var tritonTags = require('triton-tags');
+var util = require('util');
 var verror = require('verror');
 
 var common = require('./vm-common');
@@ -45,6 +46,7 @@ var PW_SUFFIX = /^(.*)_pw$/;
 var RAM_RE = /^0$|^([1-9][0-9]*$)/;
 var TRITON_TAG_ROOT_RE = /^triton\./;
 var UUID_RE = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/;
+var VOLUME_NAME_RE = /^[a-zA-Z0-9][a-zA-Z0-9_\.\-]+$/;
 
 exports.MAX_LIST_VMS_LIMIT = MAX_LIST_VMS_LIMIT;
 
@@ -255,6 +257,10 @@ var VM_FIELDS = [
         name: 'vcpus',
         mutable: false
     },
+    {
+        name: 'volumes',
+        mutable: false
+    },
     {
         name: 'zfs_data_compression',
         mutable: true
@@ -438,6 +444,8 @@ var validators = {
 
     vcpus: createValidateNumberFn('vcpus', {min: 1}),
 
+    volumes: createValidateVolumes('volumes', false),
+
     zfs_data_compression: createValidateStringFn('zfs_data_compression'),
 
     zfs_io_priority: createValidateNumberFn('zfs_io_priority'),
@@ -493,8 +501,10 @@ function createValidateMetadataFn(field) {
 /*
  * Returns a validateArray function
  */
-function createValidateArrayFn(field) {
-    return function (params) {
+function createValidateArrayFn(field, options) {
+    assert.string(field, 'field');
+
+    return function validateArray(params) {
         var errs = [];
 
         if (params[field] !== undefined && !Array.isArray(params[field])) {
@@ -892,6 +902,104 @@ function createValidateMarkerFn(field, options) {
     };
 }
 
+function validVolumeName(volumeName) {
+    if (volumeName === undefined || volumeName === null ||
+        !VOLUME_NAME_RE.test(volumeName)) {
+        return false;
+    }
+
+    return true;
+}
+
+function validVolumeMountpoint(mountpoint) {
+    if (typeof (mountpoint) !== 'string') {
+        return false;
+    }
+
+    /*
+     * Must start with '/'.
+     */
+    if (mountpoint.length === 0 || mountpoint[0] !== '/') {
+        return false;
+    }
+
+    /*
+     * Must not contain the null character.
+     */
+    if (mountpoint.indexOf('\0') !== -1) {
+        return false;
+    }
+
+    /*
+     * Must contain at least one character that is not '/'.
+     */
+    if (mountpoint.search(/[^\/]/) === -1) {
+        return false;
+    }
+
+    return true;
+}
+
+function createValidateVolumes(fieldName, required) {
+    assert.string(fieldName, 'fieldName');
+    assert.optionalBool(required, 'required');
+
+    return function validateVolumesParam(params) {
+        var errs = [];
+        var idx = 0;
+        var VALID_VOLUME_MODES = ['ro', 'rw'];
+        var volume;
+        var volumes = params[fieldName];
+
+        if (required === false && volumes === undefined) {
+            return errs;
+        }
+
+        if (!Array.isArray(volumes)) {
+            errs.push(errors.invalidParamErr(fieldName,
+                fieldName + ' must be an array of objects'));
+            return errs;
+        }
+
+        if (volumes.length === 0) {
+            errs.push(errors.invalidParamErr(fieldName,
+                fieldName + ' must be a non-empty array of objects'));
+        }
+
+        for (idx = 0; idx < volumes.length; ++idx) {
+            volume = volumes[idx];
+            if (typeof (volume) !== 'object') {
+                errs.push(errors.invalidParamErr(fieldName,
+                    fieldName + ' must be a non-empty array of objects'));
+            }
+
+            if (volume === null) {
+                errs.push(errors.invalidParamErr(fieldName,
+                    fieldName + ' must be a non-empty array of objects'));
+            }
+
+            if (volume.mode !== undefined &&
+                VALID_VOLUME_MODES.indexOf(volume.mode) === -1) {
+                errs.push(errors.invalidParamErr(fieldName,
+                    'volume mode for volume ' + util.inspect(volume) +
+                        ' must be one of ' + VALID_VOLUME_MODES.join(', ')));
+            }
+
+            if (!validVolumeName(volume.name)) {
+                errs.push(errors.invalidParamErr(fieldName,
+                    'Invalid volume name for volume ' + util.inspect(volume)));
+            }
+
+            if (!validVolumeMountpoint(volume.mountpoint)) {
+                errs.push(errors.invalidParamErr(fieldName,
+                    'Invalid volume mount point for volume ' +
+                        util.inspect(volume)));
+            }
+        }
+
+        return errs;
+    };
+}
 /*
  * Validate an individual element of the firewall_rules array
  */

lib/workflows/destroy.js

@@ -5,17 +5,21 @@
  */
 
 /*
- * Copyright (c) 2015, Joyent, Inc.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 /*
  * This is the workflow for destroying an SDC VM.
  */
 
 var async = require('async');
-var common = require('./job-common');
+
 var restify = require('restify');
-var VERSION = '7.1.0';
+
+var common = require('./job-common');
+var nfsVolumes = require('./nfs-volumes');
+
+var VERSION = '7.1.1';
 
 
 /*
@@ -206,7 +210,8 @@ var workflow = module.exports = {
         name: 'vmapi.refresh_vm',
         modules: { restify: 'restify' },
         body: common.refreshVm
-    }, {
+    },
+    {
         name: 'napi.remove_nics',
         timeout: 30,
         retry: 1,
@@ -218,6 +223,30 @@ var workflow = module.exports = {
         retry: 1,
         body: common.updateFwapi,
         modules: { sdcClients: 'sdc-clients' }
+    },
+    /*
+     * It is possible for this operation to either fail or never happen (due to
+     * the workflow job failing before getting to this task, etc.). It is not a
+     * critical problem though. Indeed, in this case, a background async process
+     * running in the VOLAPI zone will monitor VMs changing their state to
+     * 'failed' or 'deleted' to remove the corresponding references from the
+     * volumes they reference. We're still performing this operation here so
+     * that, *if possible*, the references from a VM to a volume are removed as
+     * soon as the corresponding user command (e.g "docker rm
+     * mounting-container") completes. This also explains the short timeout: if
+     * this task would slow down a destroy job by more than 10 seconds, then we
+     * timeout the whole job instead so that users can get a response back
+     * quicker. We also add this task at the end of the tasks chain (more
+     * precisely before the release ticket task, but that one runs on error) so
+     * that all other tasks have a chance to run, regardless of whether this one
+     * succeeds or not.
+     */
+    {
+        name: 'volapi.remove_volumes_references',
+        timeout: 10,
+        retry: 1,
+        body: nfsVolumes.removeVolumesReferences,
+        modules: { sdcClients: 'sdc-clients', vasync: 'vasync' }
     }, {
         name: 'cnapi.release_vm_ticket',
         timeout: 60,

lib/workflows/job-common.js

@@ -35,7 +35,15 @@ function validateForZoneAction(job, cb) {
     cb(null, 'All parameters OK!');
 }
 
+/*
+ * Clears the "skip_zone_action" flag on the job "job" so that subsequent tasks
+ * in a given workflow do not skip any action accidentally.
+ */
+function clearSkipZoneAction(job, cb) {
+    delete job.params.skip_zone_action;
 
+    cb();
+}
 
 /*
  * General purpose function to call a CNAPI endpoint. endpoint and requestMethod
@@ -1594,5 +1602,6 @@ module.exports = {
     releaseVMTicket: releaseVMTicket,
     acquireAllocationTicket: acquireAllocationTicket,
     waitOnAllocationTicket: waitOnAllocationTicket,
-    releaseAllocationTicket: releaseAllocationTicket
+    releaseAllocationTicket: releaseAllocationTicket,
+    clearSkipZoneAction: clearSkipZoneAction
 };