Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

OS-1746,OS-1739 cleanup some issues with provisioning transitions

OS-1746 have vmadmd and vmadm both watching for provisioning -> running transitions
OS-1739 remove support for ancient datasets and the joyent brand since they refuse to run mdata:exec
  • Loading branch information...
commit 927cfb6b3b79211f62655cf3b55b4e0e4466586e 1 parent c56350e
@joshwilsdon joshwilsdon authored
Showing with 448 additions and 260 deletions.
  1. +307 −216 src/vm/node_modules/VM.js
  2. +141 −44 src/vm/sbin/vmadmd.js
View
523 src/vm/node_modules/VM.js
@@ -5109,16 +5109,6 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
var loop_interval = 3; // seconds
var ival_handle;
- if (!cb) {
- // if no cb() is passed in just log if there was an error
- cb = function (err) {
- if (err) {
- VM.log.warn(err, 'cb() not passed, ignoring error: '
- + err.message);
- }
- };
- }
-
if (!BRAND_OPTIONS[vmobj.brand].features.wait_for_hwsetup) {
// do nothing for zones where we don't wait for hwsetup
cb(new Error('brand ' + vmobj.brand + ' does not support hwsetup'));
@@ -5134,14 +5124,19 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
loop_interval = options.interval;
}
+ VM.log.debug('setting hwsetup interval ' + vmobj.uuid);
ival_handle = setInterval(function () {
VM.load(vmobj.uuid, function (err, obj) {
var timeout_remaining;
+ var ival = ival_handle;
function done() {
if (ival_handle) {
- clearInterval(ival_handle);
- ival_handle = null;
+ VM.log.debug('clearing hwsetup interval ' + vmobj.uuid);
+ clearInterval(ival);
+ ival = null;
+ } else {
+ VM.log.debug('done but no hwsetup interval ' + vmobj.uuid);
}
}
@@ -5149,8 +5144,8 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
// If the VM was deleted between calls, nothing much we can do.
VM.log.error(err, 'Unable to load ' + vmobj.uuid + ' '
+ err.message);
- cb(err);
done();
+ cb(err);
return;
}
@@ -5167,11 +5162,13 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
if (timeout_remaining <= 0) {
// IMPORTANT: this may run multiple times, must be idempotent
- markVMFailure(vmobj, function (mark_err) {
+ VM.log.warn('Marking VM ' + vmobj.uuid + ' as "failed" because'
+ + ' timeout expired and we are still "provisioning"');
+ VM.markVMFailure(vmobj, function (mark_err) {
VM.log.warn(mark_err, 'zoneinit failed, zone is '
+ 'being stopped for manual investigation.');
- cb();
done();
+ cb();
});
return;
}
@@ -5186,7 +5183,7 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
if (result) {
VM.log.debug('QMP says VM ' + vmobj.uuid
+ ' completed hwsetup');
- unsetTransition(vmobj, function (unset_err) {
+ VM.unsetTransition(vmobj, function (unset_err) {
var provisioning;
var provision_success;
@@ -5206,21 +5203,16 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
function (e) {
if (e) {
- if (e.code !== 'ENOENT') {
- VM.log.error(e);
- cb(err);
- } else {
+ if (e.code === 'ENOENT') {
VM.log.debug(e);
- // in this case the file doesn't exist so
- // either moved it already from outside or
- // deleted the machine, either way the next
- // run of the Interval loop should fix.
+ } else {
+ VM.log.error(e);
}
- return;
}
- cb();
done();
+ cb();
+ return;
});
});
}
@@ -5230,13 +5222,16 @@ function markProvisionedWhenHWSetup(vmobj, options, cb)
}
// m argument should have zonename and uuid members.
-function markVMFailure(m, cb)
+exports.markVMFailure = function (m, cb)
{
if (!m || !m.hasOwnProperty('uuid') || !m.hasOwnProperty('zonename')) {
cb(new Error('markVMFailure needs uuid + zonename'));
return;
}
+ ensureLogging('markVMFailure', true);
+
+ // note: if the zone is not running this returns empty but still exit's 0
execFile('/usr/bin/ptree', ['-z', m.zonename],
function (ptree_err, stdout, stderr) {
var zcfg;
@@ -5265,7 +5260,7 @@ function markVMFailure(m, cb)
}
// attempt to remove transition
- unsetTransition(m, function (unset_err) {
+ VM.unsetTransition(m, function (unset_err) {
if (unset_err) {
VM.log.error(unset_err);
}
@@ -5284,25 +5279,7 @@ function markVMFailure(m, cb)
});
}
);
-}
-
-function waitForZoneinit(payload, cb)
-{
- VM.waitForZoneState(payload, 'running', {}, function (error) {
- if (error) {
- markVMFailure(payload, function (err) {
- VM.log.warn(err, 'zoneinit failed, zone is being '
- + 'stopped for manual investigation.');
- cb(err);
- });
- } else {
- // we're not still provisioning
- unsetTransition(payload, function (err) {
- cb(err);
- });
- }
- });
-}
+};
function svccfg(zonepath, args, callback)
{
@@ -5344,12 +5321,13 @@ exports.waitForProvisioning = function (vmobj, cb)
function done() {
if (timeout) {
- VM.log.debug('clearing timeout for ' + vmobj.uuid);
+ VM.log.debug('clearing provision timeout for ' + vmobj.uuid);
clearTimeout(timeout);
timeout = null;
}
if (watcher) {
- VM.log.debug('closing watcher for ' + vmobj.uuid);
+ VM.log.debug('closing /var/svc/provisioning watcher for '
+ + vmobj.uuid);
watcher.close();
watcher = null;
}
@@ -5378,24 +5356,33 @@ exports.waitForProvisioning = function (vmobj, cb)
'now': Date.now(0)
}, 'waiting ' + timeout_remaining + ' sec(s) for provisioning');
+ VM.log.debug('setting provision timeout for ' + vmobj.uuid);
timeout = setTimeout(function () {
- markVMFailure(vmobj, function (err) {
- var errstr = 'timed out waiting for /var/svc/provisioning to move';
+ VM.log.warn('Marking VM ' + vmobj.uuid + ' as a "failure" because we '
+ + 'hit waitForProvisioning() timeout.');
+ VM.markVMFailure(vmobj, function (err) {
+ var errstr = 'timed out waiting for /var/svc/provisioning to move '
+ + ' for ' + vmobj.uuid;
if (err) {
VM.log.warn(err, 'markVMFailure(): ' + err.message);
}
- cb(new Error(errstr));
VM.log.error(errstr);
done();
+ cb(new Error(errstr));
});
}, (timeout_remaining * 1000));
- VM.log.debug('created timeout for ' + vmobj.uuid);
-
// this starts a loop that will move provisioning -> provision_success when
// the hardware of the VM has been initialized the first time.
if (BRAND_OPTIONS[vmobj.brand].features.wait_for_hwsetup) {
- markProvisionedWhenHWSetup(vmobj);
+ markProvisionedWhenHWSetup(vmobj, {}, function (err) {
+ if (err) {
+ VM.log.error(err, 'error in markProvisionedWhenHWSetup()');
+ }
+ done();
+ cb(err);
+ });
+ return;
}
watcher = fs.watch(filename, function (evt, file) {
@@ -5406,12 +5393,16 @@ exports.waitForProvisioning = function (vmobj, cb)
fs.exists(filename, function (exists) {
if (exists) {
// somehow we still have /var/svc/provisioning!
- markVMFailure(vmobj, function (err) {
+ VM.log.warn('Marking VM ' + vmobj.uuid + ' as a "failure"'
+ + ' because we still have /var/svc/provisioning after '
+ + 'rename');
+ VM.markVMFailure(vmobj, function (err) {
if (err) {
VM.log.warn(err, 'markVMFailure(): ' + err.message);
}
- cb(new Error('provisioning exists after rename!'));
done();
+ cb(new Error('/var/svc/provisioning exists after '
+ + 'rename!'));
});
return;
}
@@ -5419,9 +5410,9 @@ exports.waitForProvisioning = function (vmobj, cb)
// So long as /var/svc/provisioning is gone, we don't care what
// replaced it. Success or failure of user script doesn't
// matter for the state, it's provisioned now.
- unsetTransition(vmobj, function (err) {
- cb();
+ VM.unsetTransition(vmobj, function (err) {
done();
+ cb();
});
return;
});
@@ -5429,14 +5420,12 @@ exports.waitForProvisioning = function (vmobj, cb)
});
VM.log.debug('created watcher for ' + vmobj.uuid);
-
};
// create and install a 'joyent' or 'kvm' brand zone.
function installZone(payload, callback)
{
var receiving = false;
- var var_svc_provisioning = false;
var vmobj;
var zoneinit = {};
@@ -5561,7 +5550,7 @@ function installZone(payload, callback)
}, function (cb) {
// if we were receiving, we're done receiving now
if (receiving) {
- unsetTransition(vmobj, cb);
+ VM.unsetTransition(vmobj, cb);
} else {
cb();
}
@@ -5580,69 +5569,24 @@ function installZone(payload, callback)
filename = path.join(vmobj.zonepath, 'root',
'/var/zoneinit/zoneinit.json');
- if (BRAND_OPTIONS[vmobj.brand].features.zoneinit) {
- fs.readFile(filename, function (error, data) {
- if (error && (error.code === 'ENOENT')) {
- // doesn't exist, leave empty
- VM.log.debug('zoneinit.json does not exist.');
- cb();
- } else if (error) {
- // error reading: fail.
- cb(error);
- } else {
- // success try to load json
- try {
- zoneinit = JSON.parse(data.toString());
- VM.log.debug({'zoneinit_json': zoneinit},
- 'parsed zoneinit.json');
- } catch (e) {
- zoneinit = {};
- VM.log.error(e);
- }
- cb();
- }
- });
- } else {
- VM.log.debug('brand does not support '
- + '/var/zoneinit/zoneinit.json, not trying to load.');
- cb();
- }
- }, function (cb) {
- // For joyent and joyent-minimal at least, set the timeout for the
- // svc start method to the value specified in the payload, or a
- // default.
-
- var timeout;
+ getZoneinitJSON(vmobj.zonepath, function (zoneinit_err, data) {
- if (BRAND_OPTIONS[vmobj.brand].features.update_mdata_exec_timeout) {
+ if (zoneinit_err) {
+ // NOTE: not existing is not going to give us a zoneinit_err
+ VM.log.warn(zoneinit_err, 'error in getZoneinitJSON');
+ cb(zoneinit_err);
+ return;
+ }
- if (payload.hasOwnProperty('mdata_exec_timeout')) {
- timeout = payload.mdata_exec_timeout;
+ if (data) {
+ zoneinit = data;
} else {
- timeout = DEFAULT_MDATA_TIMEOUT;
+ zoneinit = {};
}
- svccfg(vmobj.zonepath, [
- '-s', 'svc:/smartdc/mdata:execute',
- 'setprop', 'start/timeout_seconds', '=', 'count:', timeout
- ], function (error, stdio) {
-
- if (error) {
- VM.log.error(error, 'failed to set mdata:exec timeout');
- cb(error);
- return;
- }
-
- cb();
- });
- } else {
cb();
- }
-
+ });
}, function (cb) {
-
- var filename_1_6_x;
- var filename_1_8_x;
// var_svc_provisioning is at installZone() scope
// If we're not receiving, we're provisioning a new VM and in that
@@ -5660,64 +5604,51 @@ function installZone(payload, callback)
return;
}
- if (BRAND_OPTIONS[vmobj.brand].features.var_svc_provisioning) {
- // these brands always handle /var/svc/provisioning on this
- // platform.
+ fs.writeFile(path.join(vmobj.zonepath, 'root',
+ '/var/svc/provisioning'), '', function (err, result) {
- var_svc_provisioning = true;
- } else {
- if (zoneinit.hasOwnProperty('features')) {
- if (zoneinit.features.var_svc_provisioning) {
- VM.log.debug('features.var_svc_provisioning == true');
- var_svc_provisioning = true;
- }
+ if (err) {
+ VM.log.error(err, 'failed to create '
+ + '/var/svc/provisioning: ' + err.message);
} else {
- // Didn't load zoneinit features, so check for datasets that
- // have 04-mdata.sh. For 1.6.x datasets this was in /root
- // but in 1.8.0 and 1.8.1 it is in /var/zoneinit. For 1.8.2
- // and later we'll not get here as the zoneinit.json will
- // exist and we'll use that.
- filename_1_6_x = path.join(vmobj.zonepath,
- '/root/root/zoneinit.d/04-mdata.sh');
- filename_1_8_x = path.join(vmobj.zonepath,
- '/root/var/zoneinit/includes/04-mdata.sh');
- if (fs.existsSync(filename_1_6_x)) {
- VM.log.debug('/root/zoneinit.d/04-mdata.sh exists');
- var_svc_provisioning = true;
- } else {
- VM.log.debug('/root/zoneinit.d/04-mdata.sh does not '
- + 'exist');
- if (fs.existsSync(filename_1_8_x)) {
- VM.log.debug('/var/zoneinit/includes/04-mdata.sh '
- + 'exists');
- var_svc_provisioning = true;
- } else {
- VM.log.debug('/var/zoneinit/includes/04-mdata.sh '
- + 'does not exist');
- }
- }
+ VM.log.debug('created /var/svc/provisioning in '
+ + path.join(vmobj.zonepath, 'root'));
}
- }
- if (var_svc_provisioning) {
- fs.writeFile(path.join(vmobj.zonepath, 'root',
- '/var/svc/provisioning'), '', function (err, result) {
+ cb(err);
+ });
+ }, function (cb) {
+ // For joyent and joyent-minimal at least, set the timeout for the
+ // svc start method to the value specified in the payload, or a
+ // default.
- if (err) {
- VM.log.error(err, 'failed to create '
- + '/var/svc/provisioning: ' + err.message);
- } else {
- VM.log.debug('created /var/svc/provisioning in '
- + path.join(vmobj.zonepath, 'root'));
+ var timeout;
+
+ if (BRAND_OPTIONS[vmobj.brand].features.update_mdata_exec_timeout) {
+
+ if (payload.hasOwnProperty('mdata_exec_timeout')) {
+ timeout = payload.mdata_exec_timeout;
+ } else {
+ timeout = DEFAULT_MDATA_TIMEOUT;
+ }
+
+ svccfg(vmobj.zonepath, [
+ '-s', 'svc:/smartdc/mdata:execute',
+ 'setprop', 'start/timeout_seconds', '=', 'count:', timeout
+ ], function (error, stdio) {
+
+ if (error) {
+ VM.log.error(error, 'failed to set mdata:exec timeout');
+ cb(error);
+ return;
}
- cb(err);
+ cb();
});
} else {
- VM.log.debug('VM does not support /var/svc/provisioning, '
- + 'not creating');
cb();
}
+
}, function (cb) {
// This writes out the 'zoneconfig' file used by zoneinit to root's
// home directory in the zone.
@@ -5790,7 +5721,7 @@ function installZone(payload, callback)
// from provisioning -> either provision_success, or
// provision_failure.
- if (var_svc_provisioning && vmobj.state === 'provisioning') {
+ if (vmobj.state === 'provisioning') {
// wait for /var/svc/provisioning -> provision_success/failure
VM.waitForProvisioning(vmobj, function (err) {
@@ -5800,7 +5731,7 @@ function installZone(payload, callback)
VM.log.info('provisioning complete: '
+ '/var/svc/provisioning is gone');
// this will clear the provision transition
- unsetTransition(vmobj, function (unset_err) {
+ VM.unsetTransition(vmobj, function (unset_err) {
cb(unset_err);
});
} else {
@@ -5810,14 +5741,9 @@ function installZone(payload, callback)
}
});
} else {
- // won't have /var/svc/provision_success, so do things the old
- // way and just wait for the zone to go running (meaning it
- // rebooted after zoneinit).
-
- waitForZoneinit(payload, function (err) {
- VM.log.info(err, 'waited for zoneinit');
- cb(err);
- });
+ VM.log.info('state for ' + vmobj.uuid + ' is ' + vmobj.state
+ + ', not waiting for provisioning');
+ cb();
}
}], function (error) {
@@ -5826,6 +5752,144 @@ function installZone(payload, callback)
);
}
+function getZoneinitJSON(rootpath, cb)
+{
+ var filename;
+
+ filename = path.join(rootpath, 'root', '/var/zoneinit/zoneinit.json');
+
+ fs.readFile(filename, function (error, data) {
+ var zoneinit;
+
+ if (error && (error.code === 'ENOENT')) {
+ // doesn't exist, leave empty
+ VM.log.debug('zoneinit.json does not exist.');
+ cb();
+ } else if (error) {
+ // error reading: fail.
+ cb(error);
+ } else {
+ // success try to load json
+ try {
+ zoneinit = JSON.parse(data.toString());
+ VM.log.debug({'zoneinit_json': zoneinit},
+ 'parsed zoneinit.json');
+ cb(null, zoneinit);
+ } catch (e) {
+ cb(e);
+ }
+ }
+ });
+}
+
+function getDatasetMountpoint(dataset, callback)
+{
+ var args;
+ var cmd = '/usr/sbin/zfs';
+ var mountpoint;
+
+ args = ['get', '-H', '-o', 'value', 'mountpoint', dataset];
+
+ VM.log.debug(cmd + ' ' + args.join(' '));
+ execFile(cmd, args, function (error, stdout, stderr) {
+ if (error) {
+ VM.log.error(error, 'zfs get failed with: ' + stderr);
+ callback(error);
+ } else {
+ mountpoint = stdout.replace(/\n/g, '');
+ VM.log.debug('mountpoint: "' + mountpoint + '"');
+ callback(null, mountpoint);
+ }
+ });
+}
+
+function checkDatasetProvisionable(payload, callback)
+{
+ var dataset;
+
+ if (BRAND_OPTIONS[payload.brand].features.var_svc_provisioning) {
+ // when the brand always supports /var/svc/provisioning we don't have to
+ // worry about the dataset not supporting it.
+ callback(true);
+ return;
+ }
+
+ if (!payload.hasOwnProperty('zpool')
+ || !payload.hasOwnProperty('image_uuid')) {
+
+ VM.log.error('missing properties required to find dataset: '
+ + JSON.stringify(payload));
+ callback(false);
+ return;
+ }
+
+ dataset = payload.zpool + '/' + payload.image_uuid;
+
+ getDatasetMountpoint(dataset, function (dataset_err, mountpoint) {
+ if (dataset_err) {
+ VM.log.error('unable to find mount point for ' + dataset);
+ callback(false);
+ return;
+ }
+
+ getZoneinitJSON(dataset, function (zoneinit_err, zoneinit) {
+ var filename_1_6_x;
+ var filename_1_8_x;
+
+ if (zoneinit_err) {
+ VM.log.error(zoneinit_err, 'getZoneinitJSON() failed, assuming '
+ + 'not provisionable.');
+ callback(false);
+ return;
+ } else if (!zoneinit) {
+ VM.log.debug('no data from getZoneinitJSON(), using {}');
+ zoneinit = {};
+ }
+
+ if (zoneinit.hasOwnProperty('features')) {
+ if (zoneinit.features.var_svc_provisioning) {
+ VM.log.info('zoneinit.features.var_svc_provisioning is '
+ + 'set.');
+ callback(true);
+ return;
+ }
+ // we have features but not var_svc_provisioning === true means
+ // we can't provision. Fall through and return false.
+ } else {
+ // Didn't load zoneinit features, so check for datasets that
+ // have // 04-mdata.sh. For 1.6.x and earlier datasets this was
+ // in /root but in 1.8.0 and 1.8.1 it is in /var/zoneinit. For
+ // 1.8.2 and later we'll not get here as the zoneinit.json will
+ // exist and we'll use that.
+ filename_1_6_x = path.join(mountpoint, 'root',
+ '/root/zoneinit.d/04-mdata.sh');
+ filename_1_8_x = path.join(mountpoint, 'root',
+ '/var/zoneinit/includes/04-mdata.sh');
+
+ if (fs.existsSync(filename_1_6_x)) {
+ VM.log.info(filename_1_6_x + ' exists');
+ callback(true);
+ return;
+ } else {
+ VM.log.debug(filename_1_6_x + ' does not exist');
+ if (fs.existsSync(filename_1_8_x)) {
+ VM.log.info(filename_1_8_x + ' exists');
+ callback(true);
+ return;
+ } else {
+ VM.log.debug(filename_1_8_x + ' does not exist');
+ // this was our last chance.
+ // Fall through and return false.
+ }
+ }
+ }
+
+ callback(false);
+ return;
+ });
+ });
+}
+
// create and install a 'joyent' or 'kvm' brand zone.
function createZone(payload, callback)
{
@@ -6257,29 +6321,47 @@ function buildTransitionZonecfg(transition, target, timeout)
return cmdline;
}
-function unsetTransition(m, callback)
+exports.unsetTransition = function (m, callback)
{
zonecfg(['-u', m.uuid, 'remove attr name=transition'], function (err, fds) {
if (err) {
- VM.log.error('unable to remove transition for zone '
- + m.uuid + ' stderr: ' + fds.stderr, err);
+ // log at info because this might be because already removed
+ VM.log.info(err, 'unable to remove transition for zone '
+ + m.uuid + ' stderr: ' + fds.stderr);
}
- // We don't error here because we want to be idempotent.
- if (m.transition_to && m.transition_to === 'start') {
- VM.log.debug('vm was stopping for reboot, '
- + 'transitioning to start.');
- VM.start(m.uuid, {}, function (e) {
- if (e) {
- VM.log.error('failed to start when clearing '
- + 'transition', e);
- }
- });
- }
+ zonecfg(['-u', m.uuid, 'info attr name=transition'],
+ function (info_err, info_fds) {
- callback();
+ if (info_err) {
+ VM.log.err(info_err, 'failed to confirm transition removal');
+ callback(info_err);
+ return;
+ }
+
+ if (info_fds.stdout !== 'No such attr resource.\n') {
+ callback(new Error('transition does not appear to have been '
+ + 'removed zonecfg said: ' + JSON.stringify(info_fds)));
+ return;
+ }
+
+ // removed the transition, now attempt to start if we're rebooting.
+ if (m.transition_to && m.transition_to === 'start') {
+ VM.log.debug('VM ' + m.uuid + ' was stopping for reboot, '
+ + 'transitioning to start.');
+ VM.start(m.uuid, {}, function (e) {
+ if (e) {
+ VM.log.error(e, 'failed to start when clearing '
+ + 'transition');
+ }
+ callback();
+ });
+ } else {
+ callback();
+ }
+ });
});
-}
+};
function setTransition(m, transition, target, timeout, callback)
{
@@ -6291,7 +6373,7 @@ function setTransition(m, transition, target, timeout, callback)
async.series([
function (cb) {
if (m.hasOwnProperty('transition')) {
- unsetTransition(m, cb);
+ VM.unsetTransition(m, cb);
} else {
cb();
}
@@ -6835,30 +6917,39 @@ exports.create = function (payload, callback)
VM.log.debug('normalized payload:\n'
+ JSON.stringify(payload, null, 2));
- createZoneUUID(payload, function (e, uuid) {
- if (e) {
- callback(e);
- } else {
- if (BRAND_OPTIONS[payload.brand].features.type === 'KVM') {
- createVM(payload, function (error, result) {
- if (error) {
- callback(error);
- } else {
- callback(null, {'uuid': payload.uuid,
- 'zonename': payload.zonename});
- }
- });
+ checkDatasetProvisionable(payload, function (provisionable) {
+ if (!provisionable) {
+ VM.log.error('checkDatasetProvisionable() says dataset is '
+ + 'unprovisionable');
+ callback(new Error('provisioning dataset ' + payload.image_uuid
+ + ' with brand ' + payload.brand + ' is not supported'));
+ return;
+ }
+ createZoneUUID(payload, function (e, uuid) {
+ if (e) {
+ callback(e);
} else {
- createZone(payload, function (error, result) {
- if (error) {
- callback(error);
- } else {
- callback(null, {'uuid': payload.uuid,
- 'zonename': payload.zonename});
- }
- });
+ if (BRAND_OPTIONS[payload.brand].features.type === 'KVM') {
+ createVM(payload, function (error, result) {
+ if (error) {
+ callback(error);
+ } else {
+ callback(null, {'uuid': payload.uuid,
+ 'zonename': payload.zonename});
+ }
+ });
+ } else {
+ createZone(payload, function (error, result) {
+ if (error) {
+ callback(error);
+ } else {
+ callback(null, {'uuid': payload.uuid,
+ 'zonename': payload.zonename});
+ }
+ });
+ }
}
- }
+ });
});
});
};
@@ -8237,7 +8328,7 @@ function kill(uuid, callback)
if (msg.match(/zone is already halted$/)) {
// remove transition marker, since vm is not running now.
- unsetTransition(obj, function () {
+ VM.unsetTransition(obj, function () {
callback(null, 'vm is not running');
});
} else if (e) {
@@ -8255,7 +8346,7 @@ function kill(uuid, callback)
}
if (obj.state === 'stopping') {
// remove transition marker
- unsetTransition(obj, function () {
+ VM.unsetTransition(obj, function () {
callback(null, msg);
});
} else {
View
185 src/vm/sbin/vmadmd.js
@@ -46,7 +46,7 @@ var util = require('util');
var VMADMD_PORT = 8080;
var VMADMD_AUTOBOOT_FILE = '/tmp/.autoboot_vmadmd';
-var PROV_IVAL = {};
+var PROV_WAIT = {};
var SDC = {};
var SPICE = {};
var TIMER = {};
@@ -300,6 +300,109 @@ function loadConfig(callback)
});
}
+/*
+ * This calls cb when either the VM has finished provisioning or there was an
+ * urecoverable error.
+ *
+ * cb() will be called with:
+ *
+ * (err) -- an error object when we can't continue
+ * (null, 'success') -- when the provision succeeded
+ * (null, 'failure') -- when the provision failed or timed out
+ *
+ * NOTE:
+ * - when the provision succeeds for KVM: this will start the VNC
+ * - when the provision fails: calls VM.markVMFailrure() to put it in 'failed'
+ *
+ */
+function handleProvisioning(vmobj, cb)
+{
+ var provision_fail_fn;
+ var provision_ok_fn;
+ var provisioning_fn;
+
+ // assert vmobj.state === 'provisioning'
+
+ function success() {
+ if (vmobj.brand === 'kvm') {
+ // reload the VM to see if we should setup VNC, etc.
+ VM.load(vmobj.uuid, function (load_err, obj) {
+ if (load_err) {
+ log.error(load_err, 'unable to load VM after '
+ + 'waiting for provision: ' + load_err.message);
+ cb(load_err);
+ return;
+ }
+ log.debug('VM state is ' + obj.state + ' after provisioning');
+ if (obj.state === 'running') {
+ // clear any old timers or VNC/SPICE since this vm just came
+ // up (state was provisioning), then spin up a new VNC.
+ clearVM(obj.uuid);
+ spawnRemoteDisplay(obj);
+ }
+ cb(null, 'success');
+ });
+ } else {
+ cb(null, 'success');
+ }
+ }
+
+ function failure() {
+ VM.markVMFailure(vmobj, function (mark_err) {
+ VM.log.warn(mark_err, 'provisioning failed, zone is being stopped '
+ + 'for manual investigation.');
+ cb(null, 'failure');
+ });
+ }
+
+ provision_fail_fn = path.join(vmobj.zonepath,
+ 'root/var/svc/provision_failure');
+ provision_ok_fn = path.join(vmobj.zonepath,
+ 'root/var/svc/provision_success');
+ provisioning_fn = path.join(vmobj.zonepath,
+ 'root/var/svc/provisioning');
+
+ if (fs.existsSync(provisioning_fn)) {
+ // wait for /var/svc/provisioning to disappear
+ VM.waitForProvisioning(vmobj, function (wait_err) {
+ VM.log.debug(wait_err, 'waited for provisioning');
+ if (wait_err) {
+ cb(wait_err);
+ return;
+ }
+ success();
+ });
+ } else if (fs.existsSync(provision_ok_fn)) {
+ // no /var/svc/provisioning file, but we have success file
+ VM.unsetTransition(vmobj, function (unset_err) {
+ if (unset_err) {
+ cb(unset_err);
+ return;
+ }
+ log.info(unset_err, 'unset "provisioning" because we saw '
+ + 'provision_success for ' + vmobj.uuid);
+ success();
+ });
+ } else if (fs.existsSync(provision_fail_fn)) {
+ // we failed but someone forgot to set the flag (state==provisioning)
+ VM.log.warn('Marking VM ' + vmobj.uuid + ' as a "failure" because '
+ + provision_fail_fn + ' exists.');
+ failure();
+ } else {
+ // none of the provisioning files exist and we only support zones which
+ // are going to handle these, so we must have succeeded and just missed
+ // clearing the provisioning state.
+ log.warn('all provisioning files missing, assuming provision success.');
+ VM.unsetTransition(vmobj, function (unset_err) {
+ if (unset_err) {
+ cb(unset_err);
+ return;
+ }
+ success();
+ });
+ }
+}
+
// NOTE: nobody's paying attention to whether this completes or not.
function updateZoneStatus(ev)
{
@@ -329,40 +432,26 @@ function updateZoneStatus(ev)
if (vmobj.state === 'provisioning') {
// check that we're not already waiting.
- if (PROV_IVAL.hasOwnProperty(vmobj.uuid)) {
- log.trace('already looping for ' + vmobj.uuid);
+ if (PROV_WAIT.hasOwnProperty(vmobj.uuid)) {
+ log.trace('already waiting for ' + vmobj.uuid
+ + ' to leave "provisioning"');
return;
}
- // this just tells us that we've already got a watcher running
- // waiting for the /var/svc/provisioning to go away so we don't
- // start another one.
- PROV_IVAL[vmobj.uuid] = true;
-
- // wait for /var/svc/provisioning to disappear
- VM.waitForProvisioning(vmobj, function (wait_err) {
- VM.log.debug(wait_err, 'waited for provisioning');
- if (!wait_err && vmobj.brand === 'kvm') {
- // reload the VM to see if we should setup VNC, etc.
- VM.load(vmobj.uuid, function (load_err, obj) {
- if (load_err) {
- log.error(load_err, 'unable to load VM after '
- + 'waiting for provision: ' + load_err.message);
- return;
- }
- log.debug('VM state is ' + obj.state
- + ' after provisioning');
- if (obj.state === 'running') {
- // clear any old timers or VNC/SPICE since this vm
- // just came up, then spin up a new VNC.
- clearVM(obj.uuid);
- spawnRemoteDisplay(obj);
- }
- });
+ PROV_WAIT[vmobj.uuid] = true;
+ handleProvisioning(vmobj, function (prov_err, result) {
+ // this waiter finishd so we're ok with another being started.
+ delete PROV_WAIT[vmobj.uuid];
+ if (prov_err) {
+ log.error(prov_err, 'error handling provisioning state for'
+ + ' ' + vmobj.uuid + ': ' + prov_err.message);
+ return;
}
- delete PROV_IVAL[vmobj.uuid];
+ log.debug('handleProvision() for ' + vmobj.uuid + ' returned: '
+ + result);
+ return;
});
- return;
+
}
// don't handle transitions other than provisioning for non-kvm
@@ -1052,21 +1141,29 @@ function main()
if (vmobj.state === 'failed') {
log.debug('skipping failed VM ' + vmobj.uuid);
} else if (vmobj.state === 'provisioning') {
- log.debug('VM ' + vmobj.uuid + ' is in state '
- + '"provisioning"');
-
- // only start a provisioning watcher if we're not
- // already waiting.
- if (!PROV_IVAL.hasOwnProperty(vmobj.uuid)) {
- PROV_IVAL[vmobj.uuid] = true;
-
- // wait for /var/svc/provisioning to disappear
- VM.waitForProvisioning(vmobj, function (wait_err) {
- VM.log.debug(wait_err,
- 'waited for provisioning');
- delete PROV_IVAL[vmobj.uuid];
- });
+ log.debug('at vmadmd startup, VM ' + vmobj.uuid + ' is '
+ + 'in state "provisioning"');
+
+ if (PROV_WAIT.hasOwnProperty(vmobj.uuid)) {
+ log.warn('at vmadmd startup, already waiting for '
+ + '"provisioning" for ' + vmobj.uuid);
+ return;
}
+
+ PROV_WAIT[vmobj.uuid] = true;
+ // this calls the callback when we go out of
+ // provisioning one way or another.
+ handleProvisioning(vmobj, function (prov_err, result) {
+ delete PROV_WAIT[vmobj.uuid];
+ if (prov_err) {
+ log.error(prov_err, 'error handling '
+ + 'provisioning state for ' + vmobj.uuid
+ + ': ' + prov_err.message);
+ return;
+ }
+ log.debug('at vmadmd startup, handleProvision() for'
+ + ' ' + vmobj.uuid + ' returned: ' + result);
+ });
} else if (vmobj.brand === 'kvm') {
log.debug('calling loadVM(' + vmobj.uuid + ')');
loadVM(vmobj, do_autoboot);
Please sign in to comment.
Something went wrong with that request. Please try again.