Skip to content

Commit

Permalink
feat(xapi/VM_{checkpoint,snapshot}): HTTP sync hook (#6423)
Browse files Browse the repository at this point in the history
  • Loading branch information
julien-f committed Sep 26, 2022
1 parent f1ab625 commit f82eb8a
Show file tree
Hide file tree
Showing 5 changed files with 219 additions and 0 deletions.
131 changes: 131 additions & 0 deletions @xen-orchestra/xapi/docs/vm-sync-hook.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# VM Sync Hook

> This feature is currently _unstable_ and might change or be removed in the future.
>
> Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues).
> This feature is not currently supported for backups done with XO Proxy.
Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request.

A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk.

> This request will only be sent if the VM is in a running state.
## Configuration

The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`.

By default, it will be an HTTPS request on the port `1727`, on the first IP address reported by the VM.

If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=<URL>`.

To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-proxy` if relevant) configuration:

```toml
[xapiOptions]
syncHookSecret = 'unique long string to ensure the request comes from XO'
```

## Specification

XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_.

If the request fails for any reason, XO will go ahead with snapshot immediately.

```http
GET /sync HTTP/1.1
Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
```

When the snapshot is finished, another request will be sent:

```http
GET /post-sync HTTP/1.1
Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
```

The created snapshot will have the special `xo:synced` tag set to make it identifiable.

## Example server in Node

`index.cjs`:

```js
const exec = require('node:util').promisify(require('node:child_process').execFile)

const SECRET = 'unique long string to ensure the request comes from XO'

const HANDLERS = {
__proto__: null,

async '/sync'() {
// actions to do before the VM is snapshotted

// in this example, the Linux command `sync` is called:
await exec('sync')
},

async '/post-sync'() {
// actions to do after the VM is snapshotted
},
}

function checkAuthorization(req) {
try {
const { authorization } = req.headers
if (authorization !== undefined) {
const parts = authorization.split(' ')
if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') {
return Buffer.from(parts[1], 'base64').toString() === SECRET
}
}
} catch (error) {
console.warn('checkAuthorization', error)
}
return false
}

async function main() {
// generate a self-signed certificate
const [, key, cert] =
/^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s.exec(
(await exec('openssl', ['req', '-batch', '-new', '-x509', '-nodes', '-newkey', 'rsa:2048', '-keyout', '-']))
.stdout
)

const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) {
if (!checkAuthorization(req)) {
res.statusCode = 403
return res.end('Forbidden')
}

const handler = HANDLERS[req.url.split('?')[0]]
if (handler === undefined || req.method !== 'GET') {
res.statusCode = 404
return res.end('Not Found')
}

try {
await handler()

res.statusCode = 200
res.end('Ok')
} catch (error) {
console.warn(error)

if (!res.headersSent) {
res.statusCode = 500
res.write('Internal Error')
}
res.end()
}
})

await new Promise((resolve, reject) => {
server.on('close', resolve).on('error', reject).listen(1727)
})
}

main().catch(console.warn)
```
4 changes: 4 additions & 0 deletions @xen-orchestra/xapi/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ class Xapi extends Base {
constructor({
callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
maxUncoalescedVdis,
syncHookSecret,
syncHookTimeout,
vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
...opts
}) {
Expand All @@ -112,6 +114,8 @@ class Xapi extends Base {
when: { code: 'TOO_MANY_PENDING_TASKS' },
}
this._maxUncoalescedVdis = maxUncoalescedVdis
this._syncHookSecret = syncHookSecret
this._syncHookTimeout = syncHookTimeout
this._vdiDestroyRetryWhenInUse = {
...vdiDestroyRetryWhenInUse,
onRetry,
Expand Down
1 change: 1 addition & 0 deletions @xen-orchestra/xapi/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"@xen-orchestra/log": "^0.3.0",
"d3-time-format": "^3.0.0",
"golike-defer": "^0.5.1",
"http-request-plus": "^0.14.0",
"json-rpc-protocol": "^0.13.2",
"lodash": "^4.17.15",
"promise-toolbox": "^0.21.0",
Expand Down
81 changes: 81 additions & 0 deletions @xen-orchestra/xapi/vm.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

const CancelToken = require('promise-toolbox/CancelToken')
const groupBy = require('lodash/groupBy.js')
const hrp = require('http-request-plus')
const ignoreErrors = require('promise-toolbox/ignoreErrors')
const pickBy = require('lodash/pickBy.js')
const omit = require('lodash/omit.js')
Expand Down Expand Up @@ -46,6 +47,31 @@ const cleanBiosStrings = biosStrings => {
}
}

// See: https://github.com/xapi-project/xen-api/blob/324bc6ee6664dd915c0bbe57185f1d6243d9ed7e/ocaml/xapi/xapi_guest_agent.ml#L59-L81
//
// Returns <min(n)>/ip || <min(n)>/ipv4/<min(m)> || <min(n)>/ipv6/<min(m)> || undefined
// where n corresponds to the network interface and m to its IP
const IPV4_KEY_RE = /^\d+\/ip(?:v4\/\d+)?$/
const IPV6_KEY_RE = /^\d+\/ipv6\/\d+$/
function getVmAddress(networks) {
if (networks !== undefined) {
let ipv6
for (const key of Object.keys(networks).sort()) {
if (IPV4_KEY_RE.test(key)) {
return networks[key]
}

if (ipv6 === undefined && IPV6_KEY_RE.test(key)) {
ipv6 = networks[key]
}
}
if (ipv6 !== undefined) {
return ipv6
}
}
throw new Error('no VM address found')
}

async function listNobakVbds(xapi, vbdRefs) {
const vbds = []
await asyncMap(vbdRefs, async vbdRef => {
Expand Down Expand Up @@ -132,6 +158,51 @@ class Vm {
}
}

async _httpHook({ guest_metrics, power_state, tags, uuid }, pathname) {
if (power_state !== 'Running') {
return
}

let url
let i = tags.length
do {
if (i === 0) {
return
}
const tag = tags[--i]
if (tag === 'xo:notify-on-snapshot') {
const { networks } = await this.getRecord('VM_guest_metrics', guest_metrics)
url = Object.assign(new URL('https://locahost'), {
hostname: getVmAddress(networks),
port: 1727,
})
} else {
const prefix = 'xo:notify-on-snapshot='
if (tag.startsWith(prefix)) {
url = new URL(tag.slice(prefix.length))
}
}
} while (url === undefined)

url.pathname = pathname

const headers = {}
const secret = this._asyncHookSecret
if (secret !== undefined) {
headers.authorization = 'Bearer ' + Buffer.from(secret).toString('base64')
}

try {
await hrp(url, {
headers,
rejectUnauthorized: false,
timeout: this._syncHookTimeout ?? 60e3,
})
} catch (error) {
warn('HTTP hook failed', { error, url, vm: uuid })
}
}

async assertHealthyVdiChains(vmRef, tolerance = this._maxUncoalescedVdis) {
const vdiRefs = {}
;(await this.getRecords('VBD', await this.getField('VM', vmRef, 'VBDs'))).forEach(({ VDI: ref }) => {
Expand All @@ -148,6 +219,8 @@ class Vm {
async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
const vm = await this.getRecord('VM', vmRef)

await this._httpHook(vm, '/sync')

let destroyNobakVdis = false

if (ignoreNobakVdis) {
Expand All @@ -168,6 +241,9 @@ class Vm {
try {
const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef)

// detached async
this._httpHook(vm, '/post-sync').catch(noop)

// VM checkpoints are marked as templates, unfortunately it does not play well with XVA export/import
// which will import them as templates and not VM checkpoints or plain VMs
await pCatch.call(
Expand Down Expand Up @@ -544,6 +620,8 @@ class Vm {
) {
const vm = await this.getRecord('VM', vmRef)

await this._httpHook(vm, '/sync')

const isHalted = vm.power_state === 'Halted'

// requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
Expand Down Expand Up @@ -646,6 +724,9 @@ class Vm {
ref = await this.callAsync(cancelToken, 'VM.snapshot', vmRef, name_label).then(extractOpaqueRef)
} while (false)

// detached async
this._httpHook(vm, '/post-sync').catch(noop)

// VM snapshots are marked as templates, unfortunately it does not play well with XVA export/import
// which will import them as templates and not VM snapshots or plain VMs
await pCatch.call(
Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.unreleased.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
- [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409))
- [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411))
- [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416))
- [Sync hook] VM can now be notified before being snapshot, please [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/xapi/docs/vm-sync-hook.md) (PR [#6423](https://github.com/vatesfr/xen-orchestra/pull/6423))

### Bug fixes

Expand Down Expand Up @@ -40,6 +41,7 @@

- @vates/fuse-vhd major
- @xen-orchestra/backups minor
- @xen-orchestra/xapi minor
- vhd-lib minor
- xo-server-auth-saml patch
- xo-server minor
Expand Down

0 comments on commit f82eb8a

Please sign in to comment.