From 6b387cafab51de11d905aeeb491dec32200bc18a Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Thu, 15 Sep 2022 10:08:28 +0200 Subject: [PATCH 1/9] WiP: feat(xapi/VM_{checkpoint,snapshot}): HTTP sync hook --- packages/xo-server/docs/vm-sync-hook.md | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 packages/xo-server/docs/vm-sync-hook.md diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md new file mode 100644 index 00000000000..98d51bc0a95 --- /dev/null +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -0,0 +1,80 @@ +# VM Sync Hook + +> This feature is currently _unstable_ and might change or be removed in the future. +> +> Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues). + +Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request. + +A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk. + +## Configuration + +The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`. + +By default, it will be an HTTP request on the port `1727`, on the first IP address reported by the VM. + +If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=`. + +## Specification + +XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_. + +If the request fails for any reason, XO will go ahead with snapshot immediately. + +```http +GET /sync HTTP/1.1 +``` + +When the snapshot is finished, another request will be sent: + +```http +GET /post-sync HTTP/1.1 +``` + +## Example server in Node + +`index.mjs`: + +```js +const { createServer } = require('node:http') +const exec = require('node:util').promisify(require('node:child_process').exec) + +const HANDLERS = { + __proto__: null, + + async '/sync'() { + // actions to do before the VM is snapshotted + + // in this example, the Linux command `sync` is called: + await exec('sync') + }, + + async '/post-sync'() { + // actions to do after the VM is snapshotted + }, +} + +createServer(async function onRequest(req, res) { + const handler = HANDLERS[req.url.split('?')[0]] + if (handler === undefined || req.method !== 'GET') { + res.statusCode(404) + res.end('Not Found') + } + + try { + await handler() + + res.statusCode = 200 + res.end('Ok') + } catch (error) { + console.warn(error) + + if (!res.headersSent) { + res.statusCode = 500 + res.write('Internal Error') + } + res.end() + } +}).listen(1727) +``` From f21fb44e60449072ccd125e16dce9badf6b52e64 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Thu, 15 Sep 2022 11:17:16 +0200 Subject: [PATCH 2/9] minor fixes --- packages/xo-server/docs/vm-sync-hook.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md index 98d51bc0a95..498b9e0c907 100644 --- a/packages/xo-server/docs/vm-sync-hook.md +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -34,7 +34,7 @@ GET /post-sync HTTP/1.1 ## Example server in Node -`index.mjs`: +`index.cjs`: ```js const { createServer } = require('node:http') @@ -58,8 +58,8 @@ const HANDLERS = { createServer(async function onRequest(req, res) { const handler = HANDLERS[req.url.split('?')[0]] if (handler === undefined || req.method !== 'GET') { - res.statusCode(404) - res.end('Not Found') + res.statusCode = 404 + return res.end('Not Found') } try { From edf909ec66814aebbb406b83b507a84c576133e9 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Sat, 17 Sep 2022 13:36:14 +0200 Subject: [PATCH 3/9] HTTPS by default --- packages/xo-server/docs/vm-sync-hook.md | 56 +++++++++++++++---------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md index 498b9e0c907..c0127f1adb4 100644 --- a/packages/xo-server/docs/vm-sync-hook.md +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -12,7 +12,7 @@ A typical use case is to make sure the VM is in a consistent state during the sn The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`. -By default, it will be an HTTP request on the port `1727`, on the first IP address reported by the VM. +By default, it will be an HTTPS request on the port `1727`, on the first IP address reported by the VM. If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=`. @@ -37,8 +37,7 @@ GET /post-sync HTTP/1.1 `index.cjs`: ```js -const { createServer } = require('node:http') -const exec = require('node:util').promisify(require('node:child_process').exec) +const exec = require('node:util').promisify(require('node:child_process').execFile) const HANDLERS = { __proto__: null, @@ -55,26 +54,41 @@ const HANDLERS = { }, } -createServer(async function onRequest(req, res) { - const handler = HANDLERS[req.url.split('?')[0]] - if (handler === undefined || req.method !== 'GET') { - res.statusCode = 404 - return res.end('Not Found') - } +async function main() { + // generate a self-signed certificate + const [, key, cert] = + /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s.exec( + (await exec('openssl', ['req', '-batch', '-new', '-x509', '-nodes', '-newkey', 'rsa:2048', '-keyout', '-'])) + .stdout + ) + + const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) { + const handler = HANDLERS[req.url.split('?')[0]] + if (handler === undefined || req.method !== 'GET') { + res.statusCode = 404 + return res.end('Not Found') + } - try { - await handler() + try { + await handler() - res.statusCode = 200 - res.end('Ok') - } catch (error) { - console.warn(error) + res.statusCode = 200 + res.end('Ok') + } catch (error) { + console.warn(error) - if (!res.headersSent) { - res.statusCode = 500 - res.write('Internal Error') + if (!res.headersSent) { + res.statusCode = 500 + res.write('Internal Error') + } + res.end() } - res.end() - } -}).listen(1727) + }) + + await new Promise((resolve, reject) => { + server.on('close', resolve).on('error', reject).listen(1727) + }) +} + +main().catch(console.warn) ``` From 7734207c2926075a7524c7ed89c33496ab3bf84c Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Sat, 17 Sep 2022 13:41:33 +0200 Subject: [PATCH 4/9] tag on snapshot --- packages/xo-server/docs/vm-sync-hook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md index c0127f1adb4..329e2e497f6 100644 --- a/packages/xo-server/docs/vm-sync-hook.md +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -32,6 +32,8 @@ When the snapshot is finished, another request will be sent: GET /post-sync HTTP/1.1 ``` +The created snapshot will have the special `xo:synced` tag set to make it identifiable. + ## Example server in Node `index.cjs`: From 8c96090457a8dfcad1cffd9620b9e7be6b715333 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Sat, 17 Sep 2022 14:00:33 +0200 Subject: [PATCH 5/9] bearer token --- packages/xo-server/docs/vm-sync-hook.md | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md index 329e2e497f6..03f34ec3e80 100644 --- a/packages/xo-server/docs/vm-sync-hook.md +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -16,6 +16,13 @@ By default, it will be an HTTPS request on the port `1727`, on the first IP addr If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=`. +To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-config` if relevant) Configuration: + +```toml +[xapiOptions] +syncHookSecret = 'unique long string to ensure the request comes from XO' +``` + ## Specification XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_. @@ -41,6 +48,8 @@ The created snapshot will have the special `xo:synced` tag set to make it identi ```js const exec = require('node:util').promisify(require('node:child_process').execFile) +const SECRET = 'unique long string to ensure the request comes from XO' + const HANDLERS = { __proto__: null, @@ -56,6 +65,21 @@ const HANDLERS = { }, } +function checkAuthorization(req) { + try { + const { authorization } = req.authorization + if (authorization !== undefined) { + const parts = authorization.split(' ') + if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') { + return Buffer.from(parts[1], 'base64').toString() === SECRET + } + } + } catch (error) { + console.warn('checkAuthorization', error) + } + return false +} + async function main() { // generate a self-signed certificate const [, key, cert] = @@ -65,6 +89,11 @@ async function main() { ) const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) { + if (!checkAuthorization(req)) { + res.statusCode = 403 + return res.end('Forbidden') + } + const handler = HANDLERS[req.url.split('?')[0]] if (handler === undefined || req.method !== 'GET') { res.statusCode = 404 From 61ae57c59258a49823b9ddca013e9191044e9663 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Mon, 19 Sep 2022 01:27:20 +0200 Subject: [PATCH 6/9] minor fixes --- packages/xo-server/docs/vm-sync-hook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md index 03f34ec3e80..4a0b68c310c 100644 --- a/packages/xo-server/docs/vm-sync-hook.md +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -16,7 +16,7 @@ By default, it will be an HTTPS request on the port `1727`, on the first IP addr If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=`. -To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-config` if relevant) Configuration: +To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-proxy` if relevant) configuration: ```toml [xapiOptions] From 6d91a446dfd1ae371ee3fa189f09f6461e09aeff Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Mon, 19 Sep 2022 01:29:25 +0200 Subject: [PATCH 7/9] authorization header --- packages/xo-server/docs/vm-sync-hook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/packages/xo-server/docs/vm-sync-hook.md index 4a0b68c310c..fc6a427571d 100644 --- a/packages/xo-server/docs/vm-sync-hook.md +++ b/packages/xo-server/docs/vm-sync-hook.md @@ -31,12 +31,14 @@ If the request fails for any reason, XO will go ahead with snapshot immediately. ```http GET /sync HTTP/1.1 +Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP ``` When the snapshot is finished, another request will be sent: ```http GET /post-sync HTTP/1.1 +Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP ``` The created snapshot will have the special `xo:synced` tag set to make it identifiable. From ab8e65b5449644fe64375560f9809992f8fc1b40 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Sat, 24 Sep 2022 01:43:01 +0200 Subject: [PATCH 8/9] implementation --- .../xapi}/docs/vm-sync-hook.md | 0 @xen-orchestra/xapi/index.js | 4 + @xen-orchestra/xapi/package.json | 1 + @xen-orchestra/xapi/vm.js | 79 +++++++++++++++++++ CHANGELOG.unreleased.md | 2 + 5 files changed, 86 insertions(+) rename {packages/xo-server => @xen-orchestra/xapi}/docs/vm-sync-hook.md (100%) diff --git a/packages/xo-server/docs/vm-sync-hook.md b/@xen-orchestra/xapi/docs/vm-sync-hook.md similarity index 100% rename from packages/xo-server/docs/vm-sync-hook.md rename to @xen-orchestra/xapi/docs/vm-sync-hook.md diff --git a/@xen-orchestra/xapi/index.js b/@xen-orchestra/xapi/index.js index 7f1a7d67654..8126066d21f 100644 --- a/@xen-orchestra/xapi/index.js +++ b/@xen-orchestra/xapi/index.js @@ -102,6 +102,8 @@ class Xapi extends Base { constructor({ callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 }, maxUncoalescedVdis, + syncHookSecret, + syncHookTimeout, vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 }, ...opts }) { @@ -112,6 +114,8 @@ class Xapi extends Base { when: { code: 'TOO_MANY_PENDING_TASKS' }, } this._maxUncoalescedVdis = maxUncoalescedVdis + this._syncHookSecret = syncHookSecret + this._syncHookTimeout = syncHookTimeout this._vdiDestroyRetryWhenInUse = { ...vdiDestroyRetryWhenInUse, onRetry, diff --git a/@xen-orchestra/xapi/package.json b/@xen-orchestra/xapi/package.json index 95265f49959..63eaaf2b8eb 100644 --- a/@xen-orchestra/xapi/package.json +++ b/@xen-orchestra/xapi/package.json @@ -26,6 +26,7 @@ "@xen-orchestra/log": "^0.3.0", "d3-time-format": "^3.0.0", "golike-defer": "^0.5.1", + "http-request-plus": "^0.14.0", "json-rpc-protocol": "^0.13.2", "lodash": "^4.17.15", "promise-toolbox": "^0.21.0", diff --git a/@xen-orchestra/xapi/vm.js b/@xen-orchestra/xapi/vm.js index 083f5a4a42f..0cde970d804 100644 --- a/@xen-orchestra/xapi/vm.js +++ b/@xen-orchestra/xapi/vm.js @@ -2,6 +2,7 @@ const CancelToken = require('promise-toolbox/CancelToken') const groupBy = require('lodash/groupBy.js') +const hrp = require('http-request-plus') const ignoreErrors = require('promise-toolbox/ignoreErrors') const pickBy = require('lodash/pickBy.js') const omit = require('lodash/omit.js') @@ -46,6 +47,37 @@ const cleanBiosStrings = biosStrings => { } } +// See: https://github.com/xapi-project/xen-api/blob/324bc6ee6664dd915c0bbe57185f1d6243d9ed7e/ocaml/xapi/xapi_guest_agent.ml#L59-L81 +// +// Returns /ip || /ipv4/ || /ipv6/ || undefined +// where n corresponds to the network interface and m to its IP +const IPV4_KEY_RE = /^\d+\/ip(?:v4\/\d+)?$/ +const IPV6_KEY_RE = /^\d+\/ipv6\/\d+$/ +function getVmAddress(networks) { + if (networks !== undefined) { + let ipv6 + for (const key of Object.keys(networks).sort()) { + if (IPV4_KEY_RE.test(key)) { + return networks[key] + } + + if (ipv6 === undefined && IPV6_KEY_RE.test(key)) { + ipv6 = networks[key] + } + } + if (ipv6 !== undefined) { + return ipv6 + } + } + throw new Error('no VM address found') +} + +function stripPrefix(string, prefix) { + if (string.startsWith(prefix)) { + return string.slice(prefix.length) + } +} + async function listNobakVbds(xapi, vbdRefs) { const vbds = [] await asyncMap(vbdRefs, async vbdRef => { @@ -132,6 +164,43 @@ class Vm { } } + async _httpHook({ guest_metrics, tags, uuid }, pathname) { + let url + let i = tags.length + do { + if (i === 0) { + return + } + const tag = tags[--i] + if (tag === 'xo:notify-on-snapshot') { + const { networks } = await this.getRecord('VM_guest_metrics', guest_metrics) + url = Object.assign(new URL('https://locahost'), { + hostname: getVmAddress(networks), + port: 1727, + }) + } else { + url = new URL(stripPrefix(tag, 'xo:notify-on-snapshot=')) + } + } while (url === undefined) + + url.pathname = pathname + + const headers = {} + const secret = this._asyncHookSecret + if (secret !== undefined) { + headers.authorization = 'Bearer ' + Buffer.from(secret).toString('base64') + } + + try { + await hrp.get(url, { + headers, + timeout: this._syncHookTimeout ?? 60e3, + }) + } catch (error) { + warn('HTTP hook failed', { error, url, vm: uuid }) + } + } + async assertHealthyVdiChains(vmRef, tolerance = this._maxUncoalescedVdis) { const vdiRefs = {} ;(await this.getRecords('VBD', await this.getField('VM', vmRef, 'VBDs'))).forEach(({ VDI: ref }) => { @@ -148,6 +217,8 @@ class Vm { async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) { const vm = await this.getRecord('VM', vmRef) + await this._httpHook(vm, '/sync') + let destroyNobakVdis = false if (ignoreNobakVdis) { @@ -168,6 +239,9 @@ class Vm { try { const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef) + // detached async + this._httpHook(vm, '/post-sync').catch(noop) + // VM checkpoints are marked as templates, unfortunately it does not play well with XVA export/import // which will import them as templates and not VM checkpoints or plain VMs await pCatch.call( @@ -544,6 +618,8 @@ class Vm { ) { const vm = await this.getRecord('VM', vmRef) + await this._httpHook(vm, '/sync') + const isHalted = vm.power_state === 'Halted' // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM @@ -646,6 +722,9 @@ class Vm { ref = await this.callAsync(cancelToken, 'VM.snapshot', vmRef, name_label).then(extractOpaqueRef) } while (false) + // detached async + this._httpHook(vm, '/post-sync').catch(noop) + // VM snapshots are marked as templates, unfortunately it does not play well with XVA export/import // which will import them as templates and not VM snapshots or plain VMs await pCatch.call( diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 8f7628502bb..a22267ec40a 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -10,6 +10,7 @@ - [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409)) - [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411)) - [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416)) +- [Sync hook] VM can now be notified before being snapshot, please [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/xapi/docs/vm-sync-hook.md) (PR [#6423](https://github.com/vatesfr/xen-orchestra/pull/6423)) ### Bug fixes @@ -40,6 +41,7 @@ - @vates/fuse-vhd major - @xen-orchestra/backups minor +- @xen-orchestra/xapi minor - vhd-lib minor - xo-server-auth-saml patch - xo-server minor From b412bc83e9405864c2d905dd42097fa21902f6da Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Sat, 24 Sep 2022 02:06:22 +0200 Subject: [PATCH 9/9] fixes --- @xen-orchestra/xapi/docs/vm-sync-hook.md | 6 +++++- @xen-orchestra/xapi/vm.js | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/@xen-orchestra/xapi/docs/vm-sync-hook.md b/@xen-orchestra/xapi/docs/vm-sync-hook.md index fc6a427571d..f8cb8c3dad0 100644 --- a/@xen-orchestra/xapi/docs/vm-sync-hook.md +++ b/@xen-orchestra/xapi/docs/vm-sync-hook.md @@ -4,10 +4,14 @@ > > Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues). +> This feature is not currently supported for backups done with XO Proxy. + Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request. A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk. +> This request will only be sent if the VM is in a running state. + ## Configuration The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`. @@ -69,7 +73,7 @@ const HANDLERS = { function checkAuthorization(req) { try { - const { authorization } = req.authorization + const { authorization } = req.headers if (authorization !== undefined) { const parts = authorization.split(' ') if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') { diff --git a/@xen-orchestra/xapi/vm.js b/@xen-orchestra/xapi/vm.js index 0cde970d804..1e54afa803f 100644 --- a/@xen-orchestra/xapi/vm.js +++ b/@xen-orchestra/xapi/vm.js @@ -72,12 +72,6 @@ function getVmAddress(networks) { throw new Error('no VM address found') } -function stripPrefix(string, prefix) { - if (string.startsWith(prefix)) { - return string.slice(prefix.length) - } -} - async function listNobakVbds(xapi, vbdRefs) { const vbds = [] await asyncMap(vbdRefs, async vbdRef => { @@ -164,7 +158,11 @@ class Vm { } } - async _httpHook({ guest_metrics, tags, uuid }, pathname) { + async _httpHook({ guest_metrics, power_state, tags, uuid }, pathname) { + if (power_state !== 'Running') { + return + } + let url let i = tags.length do { @@ -179,7 +177,10 @@ class Vm { port: 1727, }) } else { - url = new URL(stripPrefix(tag, 'xo:notify-on-snapshot=')) + const prefix = 'xo:notify-on-snapshot=' + if (tag.startsWith(prefix)) { + url = new URL(tag.slice(prefix.length)) + } } } while (url === undefined) @@ -192,8 +193,9 @@ class Vm { } try { - await hrp.get(url, { + await hrp(url, { headers, + rejectUnauthorized: false, timeout: this._syncHookTimeout ?? 60e3, }) } catch (error) {