Skip to content

Commit

Permalink
TRITON-2127 sdc-migrate fails for instances that use deletion_protect…
Browse files Browse the repository at this point in the history 
…ion (#64)
  • Loading branch information
@twhiteman
twhiteman committed May 7, 2020
1 parent fe7ee03 commit 6c57e97
Show file tree
Hide file tree
Showing 9 changed files with 459 additions and 63 deletions.
170 changes: 118 additions & 52 deletions lib/vm-migration/migrate.js
View file
Expand Up @@ -243,79 +243,145 @@ function migrationFinalize(req, migrationRecord, callback) {
var headers = {'x-request-id': req.getId()};
var timeout = 15 * 60 * 1000; // 15 minutes

function waitForTask(taskId, cb) {
// Wait for the cnapi task to finish.
req.app.cnapi.waitTask(taskId, {timeout: timeout},
function (waitErr, waitReq, waitRes, task) {
// Allow a 404 "Vm not found" error - that makes our job easier.
if (waitErr && waitErr.statusCode !== 404) {
cb(waitErr);
return;
}

if (!task) {
cb();
return;
}

if (task.status !== 'complete') {
var errEntries;
if (Array.isArray(task.history) && task.history.length) {
errEntries = task.history.filter(function _histFilt(h) {
return h.name === 'error';
});
}
if (errEntries && errEntries.length) {
errMsg += ': ' + errEntries.map(function _errMap(e) {
if (e.event && e.event.error) {
return e.event.error.message ||
String(e.event.error);
}
return String(e);
}).join(', ');
} else {
errMsg += ' (task id ' + task.id + ')';
}
cb(new restify.errors.InternalError(errMsg));
return;
}

if (!Array.isArray(task.history) ||
task.history.length === 0 ||
task.history.slice(-1)[0].name !== 'finish' ||
!task.history.slice(-1)[0].event) {
cb(new restify.errors.InternalError(
errMsg + ': no cnapi task finish event'));
return;
}

cb();
});
}

vasync.pipeline({arg: {}, funcs: [
function cnapiDeleteSourceVmRequest(ctx, cb) {
var delOpts = {
headers: headers,
path: '/servers/' + migrationRecord.source_server_uuid +
'/vms/' + migrationRecord.vm_uuid + '?include_dni=true'
};
function cnapiRemoveSourceVmIndestructibleZoneroot(ctx, cb) {
if (!req.vm.indestructible_zoneroot) {
cb();
return;
}

req.app.cnapi.client.del(delOpts,
function _cnapiDeleteCb(err, cnReq, cnRes, task) {
// Allow a 404 "Vm not found" error - that makes our job easier.
if (err && err.statusCode !== 404) {
cb(null, 'OK - target instance does not exist');
var url = '/servers/' +
migrationRecord.source_server_uuid + '/vms/' +
migrationRecord.vm_uuid + '/migrate';
var payload = {
action: 'set-indestructible-zoneroot',
migrationTask: {
action: 'finalize',
record: migrationRecord
},
vm_uuid: migrationRecord.vm_uuid,
value: 'false'
};
req.app.cnapi.client.post(url, payload,
function _setIndestructibleCb(err, cnReq, cnRes, task) {
if (err) {
// Allow a 404 "Vm not found" error.
if (err.statusCode === 404) {
cb(null, 'OK - source instance does not exist');
} else {
cb(err);
}
return;
}

ctx.task = task;
cb();
waitForTask(task.id, cb);
});
},

function cnapiWaitDeleteTask(ctx, cb) {
if (!ctx.task || !ctx.task.id) {
function cnapiRemoveSourceVmIndestructibleDelegated(ctx, cb) {
if (!req.vm.indestructible_delegated) {
cb();
return;
}

// Wait for the cnapi task to finish.
req.app.cnapi.waitTask(ctx.task.id, {timeout: timeout},
function (waitErr, waitReq, waitRes, task) {
// Allow a 404 "Vm not found" error - that makes our job easier.
if (waitErr && waitErr.statusCode !== 404) {
cb(waitErr);
var url = '/servers/' +
migrationRecord.source_server_uuid + '/vms/' +
migrationRecord.vm_uuid + '/migrate';
var payload = {
action: 'set-indestructible-delegated',
migrationTask: {
action: 'finalize',
record: migrationRecord
},
vm_uuid: migrationRecord.vm_uuid,
value: 'false'
};
req.app.cnapi.client.post(url, payload,
function _setIndestructibleCb(err, cnReq, cnRes, task) {
if (err) {
// Allow a 404 "Vm not found" error.
if (err.statusCode === 404) {
cb(null, 'OK - source instance does not exist');
} else {
cb(err);
}
return;
}

if (!task) {
cb();
return;
}
waitForTask(task.id, cb);
});
},

if (task.status !== 'complete') {
var errEntries;
if (Array.isArray(task.history) && task.history.length) {
errEntries = task.history.filter(function _histFilt(h) {
return h.name === 'error';
});
}
if (errEntries && errEntries.length) {
errMsg += ': ' + errEntries.map(function _errMap(e) {
if (e.event && e.event.error) {
return e.event.error.message ||
String(e.event.error);
}
return String(e);
}).join(', ');
function cnapiDeleteSourceVmRequest(ctx, cb) {
var delOpts = {
headers: headers,
path: '/servers/' + migrationRecord.source_server_uuid +
'/vms/' + migrationRecord.vm_uuid + '?include_dni=true'
};

req.app.cnapi.client.del(delOpts,
function _cnapiDeleteCb(err, cnReq, cnRes, task) {
if (err) {
// Allow a 404 "Vm not found" error.
if (err.statusCode === 404) {
cb(null, 'OK - source instance does not exist');
} else {
errMsg += ' (task id ' + task.id + ')';
cb(err);
}
cb(new restify.errors.InternalError(errMsg));
return;
}

if (!Array.isArray(task.history) ||
task.history.length === 0 ||
task.history.slice(-1)[0].name !== 'finish' ||
!task.history.slice(-1)[0].event) {
cb(new restify.errors.InternalError(
errMsg + ': no cnapi task finish event'));
return;
}

cb();
waitForTask(task.id, cb);
});
},

Expand Down
10 changes: 9 additions & 1 deletion lib/workflows/migrate-rollback.js
View file
Expand Up @@ -5,7 +5,7 @@
*/

/*
* Copyright (c) 2019, Joyent, Inc.
* Copyright 2020 Joyent, Inc.
*/

/*
Expand Down Expand Up @@ -70,6 +70,14 @@ var workflow = module.exports = {
rollback.tasks.removeSourceDoNotInventory,
common.tasks.waitTask,

common.tasks.setupForWaitTask,
rollback.tasks.removeTargetIndestructibleZoneroot,
common.tasks.waitTask,

common.tasks.setupForWaitTask,
rollback.tasks.removeTargetIndestructibleDelegated,
common.tasks.waitTask,

/* Destroy the target vm. */
common.tasks.setupForWaitTask,
rollback.tasks.deleteTargetDniVm,
Expand Down
11 changes: 9 additions & 2 deletions lib/workflows/migrate-switch.js
View file
Expand Up @@ -5,7 +5,7 @@
*/

/*
* Copyright (c) 2019, Joyent, Inc.
* Copyright 2020 Joyent, Inc.
*/

/*
Expand Down Expand Up @@ -88,6 +88,14 @@ var workflow = module.exports = {
modSwitch.tasks.removeTargetSnapshots,
common.tasks.waitTask,

common.tasks.setupForWaitTask,
modSwitch.tasks.restoreIndestructibleZoneroot,
common.tasks.waitTask,

common.tasks.setupForWaitTask,
modSwitch.tasks.restoreIndestructibleDelegated,
common.tasks.waitTask,

/* Set quotas back as they should be. */
common.tasks.setupForWaitTask,
migrationCommon.tasks.restoreSourceZfsQuota,
Expand All @@ -98,7 +106,6 @@ var workflow = module.exports = {
common.tasks.waitTask,

/* All done - record final details and store as successful */
// modSwitch.tasks.recordServerDetails,
migrationCommon.tasks.storeSuccess,

common.tasks.releaseVMTicket,
Expand Down
7 changes: 6 additions & 1 deletion lib/workflows/vm-migration/begin.js
View file
Expand Up @@ -5,7 +5,7 @@
*/

/*
* Copyright 2019 Joyent, Inc.
* Copyright 2020 Joyent, Inc.
*/


Expand Down Expand Up @@ -341,6 +341,11 @@ function createProvisionPayload(job, cb) {
job.log.info({cpu_type: vmPayload.cpu_type}, 'setting cpu_type');
}

// Do not have deletion_protection initially set (as this will cause
// problems during initial syncing).
delete vmPayload.indestructible_zoneroot;
delete vmPayload.indestructible_delegated;

cb(null, 'created vm migrate target payload');
}

Expand Down
102 changes: 101 additions & 1 deletion lib/workflows/vm-migration/rollback.js
View file
Expand Up @@ -5,7 +5,7 @@
*/

/*
* Copyright 2019 Joyent, Inc.
* Copyright 2020 Joyent, Inc.
*/


Expand Down Expand Up @@ -296,6 +296,86 @@ function removeSourceDoNotInventory(job, cb) {
}


function removeTargetIndestructibleZoneroot(job, cb) {
var record = job.params.migrationTask.record;

assert.object(job.params.vm, 'job.params.vm');
assert.uuid(record.target_server_uuid, 'record.target_server_uuid');
assert.uuid(record.target_vm_uuid, 'record.target_vm_uuid');

if (!job.params.vm.indestructible_zoneroot) {
job.params.skip_zone_action = true;
cb(null, 'OK - indestructible_zoneroot is not set');
return;
}

// Set indestructible_zoneroot on the target instance.
var cnapi = restify.createJsonClient({
url: cnapiUrl,
headers: { 'x-request-id': job.params['x-request-id'] }
});
var url = '/servers/' +
record.target_server_uuid + '/vms/' +
record.target_vm_uuid + '/migrate';
var payload = {
action: 'set-indestructible-zoneroot',
migrationTask: job.params.migrationTask,
vm_uuid: record.target_vm_uuid,
value: 'false'
};

cnapi.post(url, payload, function _setIndestructible(err, req, res, task) {
if (err) {
cb(err);
return;
}

job.taskId = task.id;
cb(null, 'OK - task id: ' + task.id + ' queued to CNAPI!');
});
}


function removeTargetIndestructibleDelegated(job, cb) {
var record = job.params.migrationTask.record;

assert.object(job.params.vm, 'job.params.vm');
assert.uuid(record.target_server_uuid, 'record.target_server_uuid');
assert.uuid(record.target_vm_uuid, 'record.target_vm_uuid');

if (!job.params.vm.indestructible_delegated) {
job.params.skip_zone_action = true;
cb(null, 'OK - indestructible_delegated is not set');
return;
}

// Set indestructible_delegated on the target instance.
var cnapi = restify.createJsonClient({
url: cnapiUrl,
headers: { 'x-request-id': job.params['x-request-id'] }
});
var url = '/servers/' +
record.target_server_uuid + '/vms/' +
record.target_vm_uuid + '/migrate';
var payload = {
action: 'set-indestructible-delegated',
migrationTask: job.params.migrationTask,
vm_uuid: record.target_vm_uuid,
value: 'false'
};

cnapi.post(url, payload, function _setIndestructible(err, req, res, task) {
if (err) {
cb(err);
return;
}

job.taskId = task.id;
cb(null, 'OK - task id: ' + task.id + ' queued to CNAPI!');
});
}


function updateVmServerUuid(job, cb) {
var record = job.params.migrationTask.record;

Expand Down Expand Up @@ -473,6 +553,26 @@ module.exports = {
restify: 'restify'
}
},
removeTargetIndestructibleZoneroot: {
name: 'migration.rollback.removeTargetIndestructibleZoneroot',
timeout: 300,
// retry: 1,
body: removeTargetIndestructibleZoneroot,
modules: {
assert: 'assert-plus',
restify: 'restify'
}
},
removeTargetIndestructibleDelegated: {
name: 'migration.rollback.removeTargetIndestructibleDelegated',
timeout: 300,
// retry: 1,
body: removeTargetIndestructibleDelegated,
modules: {
assert: 'assert-plus',
restify: 'restify'
}
},
setTargetDoNotInventory: {
name: 'migration.rollback.setTargetDoNotInventory',
timeout: 300,
Expand Down

0 comments on commit 6c57e97

Please sign in to comment.