VM.js

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at http://smartos.org/CDDL
 *
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file.
 *
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
 *
 * Experimental functions, expect these interfaces to be unstable and
 * potentially go away entirely:
 *
 * install(uuid, callback)
 * receive(target, options, callback)
 * send(uuid, where, options, callback)
 * getSysinfo(args, callback)
 * validate(brand, action, payload, callback)
 * waitForZoneState(payload, state, options, callback)
 *
 * Exported functions:
 *
 * console(uuid, callback)
 * create(properties, callback)
 * delete(uuid, callback)
 * flatten(vmobj, key)
 * info(uuid, types, callback)
 * load([zonename|uuid], callback)
 * lookup(match, callback)
 * reboot(uuid, options={[force=true]}, callback)
 * start(uuid, extra, callback)
 * stop(uuid, options={[force=true]}, callback)
 * sysrq(uuid, req=[nmi|screenshot], options={}, callback)
 * update(uuid, properties, callback)
 *
 * Exported variables:
 *
 * logname - you can set this to a string [a-zA-Z_] to use as log name
 * logger - you can set this to a node-bunyan log stream to capture the logs
 * logger - a function which takes (priority, message, <Error Obj>)
 * INFO_TYPES - list of supported types for the info command
 * SYSRQ_TYPES - list of supported requests for sysrq
 */

// Ensure we're using the platform's node
require('/usr/node/node_modules/platform_node_version').assert();

var assert = require('assert');
var async = require('/usr/node/node_modules/async');
var bunyan = require('/usr/node/node_modules/bunyan');
var cp = require('child_process');
var dladm = require('/usr/vm/node_modules/dladm');
var exec = cp.exec;
var execFile = cp.execFile;
var expat = require('/usr/node/node_modules/node-expat');
var fs = require('fs');
var http = require('http');
var net = require('net');
var path = require('path');
var Qmp = require('/usr/vm/node_modules/qmp').Qmp;
var spawn = cp.spawn;
var sprintf = require('/usr/node/node_modules/sprintf').sprintf;
var system = require('/usr/node/node_modules/system');
var tty = require('tty');
var util = require('util');

var log_to_file = false;

// keep the last 50 messages just in case we end up wanting them.
var ringbuffer = new bunyan.RingBuffer({ limit: 50 });

/*
 * zone states from libzonecfg/common/zonecfg_impl.h
 *
 * #define ZONE_STATE_STR_CONFIGURED       "configured"
 * #define ZONE_STATE_STR_INCOMPLETE       "incomplete"
 * #define ZONE_STATE_STR_INSTALLED        "installed"
 * #define ZONE_STATE_STR_READY            "ready"
 * #define ZONE_STATE_STR_MOUNTED          "mounted"
 * #define ZONE_STATE_STR_RUNNING          "running"
 * #define ZONE_STATE_STR_SHUTTING_DOWN    "shutting_down"
 * #define ZONE_STATE_STR_DOWN             "down"
 *
 */

exports.FLATTENABLE_ARRAY_HASH_KEYS = [
    'disks',
    'nics'
];
exports.FLATTENABLE_HASH_KEYS = [
    'customer_metadata',
    'internal_metadata',
    'tags'
];

var DEFAULT_MDATA_TIMEOUT = 300;
var PROVISION_TIMEOUT = 300;
var STOP_TIMEOUT = 60;
var VM = this;

VM.log = null;

// can be (re)set by loader before we start.
exports.logger = null;
exports.loglevel = 'debug';

// OpenOnErrorFileStream is a bunyan stream that only creates the file when
// there's an error or higher level message.  We use this for actions that
//  shouldn't log in the normal case but where we do want logs when something
// breaks.  Thanks to Trent++ for most of this code.

function OpenOnErrorFileStream(filename) {
    this.path = filename;
    this.write = this.constructor.prototype.write1;
    this.end = this.constructor.prototype.end1;
}

OpenOnErrorFileStream.prototype.end1 = function () {
    // in initial mode we're not writing anything, so nothing to flush
    return;
};

// used until first ERROR or higher, then opens file and ensures future writes
// go to .write2()
OpenOnErrorFileStream.prototype.write1 = function (rec) {
    var r;

    if (rec.level >= bunyan.ERROR || log_to_file) {
        this.stream = fs.createWriteStream(this.path,
            {flags: 'a', encoding: 'utf8'});
        this.end = this.stream.end;
        this.write = this.constructor.prototype.write2;
        // dump out logs from ringbuffer too since there was an error so we can
        // figure out what's going on.
        for (r in ringbuffer.records) {
            r = ringbuffer.records[r];
            if (r != rec) {
                this.write(r);
            }
        }

        this.write(rec);
    }
};

// used when writing to file
OpenOnErrorFileStream.prototype.write2 = function (rec) {
    var str = JSON.stringify(rec, bunyan.safeCycles()) + '\n';
    this.stream.write(str);
};

// This function should be called by any exported function from this module.
// 'action' should be the name of whatever we're trying to do and side_effects
// should be true if this action is expected to change the state or modify VMs
// on the system: eg. create, start, stop, delete and false when the action
// should not cause changes to the system, eg.: load, lookup, info, console, &c.
function ensureLogging(action, side_effects)
{
    side_effects = !!side_effects; // make it boolean (undef === false)

    var filename;
    var logname;
    var streams = [];

    // This is here in case an app calls a lookup first and then a create.  The
    // logger will get created in no-sideeffects mode for the lookup but when
    // the create is called this will force the switch to writing.
    if (side_effects) {
        log_to_file = true;
    }

    if (VM.log) {
        // We're already logging, don't break things.
        return;
    }

    if (VM.hasOwnProperty('logname')) {
        logname = VM.logname.replace(/[^a-zA-Z\_]/g, '');
    }
    if (!logname || logname.length < 1) {
        logname = 'VM';
    }

    if (VM.hasOwnProperty('logger') && VM.logger) {
        // Use concat, in case someone's sneaky and makes more than one logger.
        // We don't officially support that yet though.
        streams = streams.concat(VM.logger);
    }

    filename = '/var/log/vm/' + Date.now(0) + '-'
        + sprintf('%06d', process.pid) + '-' + logname + '.log';

    // Add the ringbuffer which we'll dump if we switch from not writing to
    // writing, and so that they'll show up in dumps.
    streams.push({
        level: 'trace',
        type: 'raw',
        stream: ringbuffer
    });

    if (side_effects) {
        streams.push({
            level: VM.loglevel,
            path: filename
        });
    } else {
        streams.push({
            type: 'raw',
            stream: new OpenOnErrorFileStream(filename),
            level: VM.loglevel
        });
    }

    VM.log = bunyan.createLogger({
        name: logname,
        streams: streams,
        serializers: bunyan.stdSerializers
    });
}

function ltrim(str, chars)
{
    chars = chars || '\\s';
    str = str || '';
    return str.replace(new RegExp('^[' + chars + ']+', 'g'), '');
}

function rtrim(str, chars)
{
    chars = chars || '\\s';
    str = str || '';
    return str.replace(new RegExp('[' + chars + ']+$', 'g'), '');
}

function trim(str, chars)
{
    return ltrim(rtrim(str, chars), chars);
}

function isUUID(str) {
    var re = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/;
    if (str.length === 36 && str.match(re)) {
        return true;
    } else {
        return false;
    }
}

function fixBoolean(str)
{
    if (str === 'true') {
        return true;
    } else if (str === 'false') {
        return false;
    } else {
        return str;
    }
}

function fixBooleanLoose(str)
{
    if (str === 'true' || str === '1' || str === 1) {
        return true;
    } else if (str === 'false' || str === '0' || str === 0) {
        return false;
    } else {
        return str;
    }
}

// IMPORTANT:
//
//  Some of these properties get translated below into backward compatible
//  names.
//

var UPDATABLE_NIC_PROPS = [
    'primary',
    'nic_tag',
    'vrrp_vrid',
    'vrrp_primary_ip',
    'blocked_outgoing_ports',
    'mac',
    'gateway',
    'ip',
    'model',
    'netmask',
    'dhcp_server',
    'allow_dhcp_spoofing',
    'allow_ip_spoofing',
    'allow_mac_spoofing',
    'allow_restricted_traffic',
    'allow_unfiltered_promisc',
    'vlan_id'
];

var UPDATABLE_DISK_PROPS = [
    'boot',
    'model'
];

// Note: this doesn't include 'state' because of 'stopping' which is a virtual
// state and therefore lookups would be wrong (because they'd search on real
// state).
var QUICK_LOOKUP = [
    'zoneid',
    'zonename',
    'zonepath',
    'uuid',
    'brand',
    'ip_type'
];

exports.DISK_MODELS = [
    'virtio',
    'ide',
    'scsi'
];

exports.VGA_TYPES = [
    'cirrus',
    'std',
    'vmware',
    'qxl',
    'xenfb'
];

exports.INFO_TYPES = [
    'all',
    'block',
    'blockstats',
    'chardev',
    'cpus',
    'kvm',
    'pci',
    'spice',
    'status',
    'version',
    'vnc'
];

exports.SYSRQ_TYPES = [
    'nmi',
    'screenshot'
];

exports.COMPRESSION_TYPES = [
    'on',
    'off',
    'lzjb',
    'gzip',
    'gzip-1',
    'gzip-2',
    'gzip-3',
    'gzip-4',
    'gzip-5',
    'gzip-6',
    'gzip-7',
    'gzip-8',
    'gzip-9',
    'zle'
];

exports.KVM_MEM_OVERHEAD = 1024;
exports.KVM_MIN_MEM_OVERHEAD = 256;

var XML_PROPERTIES = {
    'zone': {
        'name': 'zonename',
        'zonepath': 'zonepath',
        'autoboot': 'autoboot',
        'brand': 'brand',
        'limitpriv': 'limit_priv',
        'fs-allowed': 'fs_allowed'
    },
    'zone.attr': {
        'alias': 'alias',
        'billing-id': 'billing_id',
        'boot': 'boot',
        'cpu-type': 'cpu_type',
        'create-timestamp': 'create_timestamp',
        'dataset-uuid': 'image_uuid',
        'default-gateway': 'default_gateway',
        'dns-domain': 'dns_domain',
        'do-not-inventory': 'do_not_inventory',
        'failed': 'failed',
        'hostname': 'hostname',
        'never-booted': 'never_booted',
        'owner-uuid': 'owner_uuid',
        'package-name': 'package_name',
        'package-version': 'package_version',
        'qemu-extra-opts': 'qemu_extra_opts',
        'qemu-opts': 'qemu_opts',
        'ram': 'ram',
        'resolvers': 'resolvers',
        'spice-opts': 'spice_opts',
        'spice-password': 'spice_password',
        'spice-port': 'spice_port',
        'tmpfs': 'tmpfs',
        'transition': 'transition',
        'vcpus': 'vcpus',
        'vga': 'vga',
        'virtio-txtimer': 'virtio_txtimer',
        'virtio-txburst': 'virtio_txburst',
        'vm-autoboot': 'vm_autoboot',
        'vnc-password': 'vnc_password',
        'vnc-port': 'vnc_port'
    },
    'zone.rctl.zone.cpu-shares.rctl-value': {
        'limit': 'cpu_shares'
    },
    'zone.rctl.zone.cpu-cap.rctl-value': {
        'limit': 'cpu_cap'
    },
    'zone.rctl.zone.zfs-io-priority.rctl-value': {
        'limit': 'zfs_io_priority'
    },
    'zone.rctl.zone.max-lwps.rctl-value': {
        'limit': 'max_lwps'
    },
    'zone.rctl.zone.max-physical-memory.rctl-value': {
        'limit': 'max_physical_memory'
    },
    'zone.rctl.zone.max-locked-memory.rctl-value': {
        'limit': 'max_locked_memory'
    },
    'zone.rctl.zone.max-swap.rctl-value': {
        'limit': 'max_swap'
    },
    'nic': {
        'ip': 'ip',
        'mac-addr': 'mac',
        'physical': 'interface',
        'vlan-id': 'vlan_id',
        'global-nic': 'nic_tag',
        'dhcp_server': 'dhcp_server',
        'allow_dhcp_spoofing': 'allow_dhcp_spoofing',
        'allow_ip_spoofing': 'allow_ip_spoofing',
        'allow_mac_spoofing': 'allow_mac_spoofing',
        'allow_restricted_traffic': 'allow_restricted_traffic',
        'allow_unfiltered_promisc': 'allow_unfiltered_promisc',
        'netmask': 'netmask',
        'model': 'model',
        'gateway': 'gateway',
        'primary': 'primary',
        'vrrp_vrid': 'vrrp_vrid',
        'vrrp_primary_ip': 'vrrp_primary_ip',
        'blocked-outgoing-ports': 'blocked_outgoing_ports'
    },
    'filesystem': {
        'special': 'source',
        'directory': 'target',
        'type': 'type',
        'raw': 'raw'
    },
    'disk': {
        'boot': 'boot',
        'image-size': 'image_size',
        'image-name': 'image_name',
        'image-uuid': 'image_uuid',
        'match': 'path',
        'media': 'media',
        'model': 'model',
        'size': 'size'
    }
};

/*
 * This allows one to define a function that will be run over the values from
 * the zonecfg at the point where we transform that data into a VM object.
 *
 */
var XML_PROPERTY_TRANSFORMS = {
    'alias': unbase64,
    'autoboot': fixBoolean,
    'cpu_cap': numberify,
    'cpu_shares': numberify,
    'disks': {
        'boot': fixBoolean,
        'image_size': numberify,
        'size': numberify
    },
    'do_not_inventory': fixBoolean,
    'max_locked_memory': unmangleMem,
    'max_lwps': numberify,
    'max_physical_memory': unmangleMem,
    'max_swap': unmangleMem,
    'never_booted': fixBoolean,
    'nics': {
        'dhcp_server': fixBoolean,
        'allow_dhcp_spoofing': fixBoolean,
        'allow_ip_spoofing': fixBoolean,
        'allow_mac_spoofing': fixBoolean,
        'allow_restricted_traffic': fixBoolean,
        'allow_unfiltered_promisc': fixBoolean,
        'primary': fixBooleanLoose,
        'vrrp_vrid': numberify,
        'vlan_id': numberify
    },
    'qemu_extra_opts': unbase64,
    'qemu_opts': unbase64,
    'ram': numberify,
    'resolvers': separateCommas,
    'spice_password': unbase64,
    'spice_port': numberify,
    'spice_opts': unbase64,
    'tmpfs': numberify,
    'vcpus': numberify,
    'virtio_txburst': numberify,
    'virtio_txtimer': numberify,
    'vnc_password': unbase64,
    'vnc_port': numberify,
    'zfs_io_priority': numberify,
    'zoneid': numberify
};

/*
 * This defines all of the possible properties that could be in a create/update
 * payload and their types. Each of the entries are required to have at least
 * a 'type' property which is one of:
 *
 *  object-array    -- an array of objects
 *  boolean         -- true or false
 *  flat-object     -- an object that has only string properties
 *  integer         -- integers only
 *  list            -- Either comma separated or array list of strings
 *  string          -- Simple string
 *  uuid            -- A standard 00000000-0000-0000-0000-000000000000 type uuid
 *  zpool           -- The name of an existing zpool
 *
 */
var PAYLOAD_PROPERTIES = {
    'add_disks': {'type': 'object-array', 'check_as': 'disks'},
    'add_nics': {'type': 'object-array', 'check_as': 'nics'},
    'alias': {'type': 'string'},
    'autoboot': {'type': 'boolean'},
    'billing_id': {'type': 'string'},
    'boot': {'type': 'string'},
    'brand': {'type': 'string'},
    'cpu_cap': {'type': 'integer'},
    'cpu_shares': {'type': 'integer'},
    'cpu_type': {'type': 'string'},
    'create_only': {'type': 'boolean'},
    'create_timestamp': {'type': 'string'},
    'customer_metadata': {'type': 'flat-object'},
    'dataset_uuid': {'type': 'uuid'},
    'delegate_dataset': {'type': 'boolean'},
    'disks': {'type': 'object-array'},
    'disks.*.block_size': {'type': 'integer'},
    'disks.*.boot': {'type': 'boolean'},
    'disks.*.compression': {'type': 'string'},
    'disks.*.image_name': {'type': 'string'},
    'disks.*.image_size': {'type': 'integer'},
    'disks.*.image_uuid': {'type': 'uuid'},
    'disks.*.size': {'type': 'integer'},
    'disks.*.media': {'type': 'string'},
    'disks.*.model': {'type': 'string'},
    'disks.*.nocreate': {'type': 'boolean'},
    'disks.*.path': {'type': 'string'},
    'disks.*.zpool': {'type': 'zpool'},
    'disk_driver': {'type': 'string'},
    'do_not_inventory': {'type': 'boolean'},
    'dns_domain': {'type': 'string'},
    'filesystems': {'type': 'object-array'},
    'filesystems.*.type': {'type': 'string'},
    'filesystems.*.source': {'type': 'string'},
    'filesystems.*.target': {'type': 'string'},
    'filesystems.*.raw': {'type': 'string'},
    'filesystems.*.options': {'type': 'list'},
    'fs_allowed': {'type': 'list'},
    'hostname': {'type': 'string'},
    'image_uuid': {'type': 'uuid'},
    'internal_metadata': {'type': 'flat-object'},
    'limit_priv': {'type': 'list'},
    'max_locked_memory': {'type': 'integer'},
    'max_lwps': {'type': 'integer'},
    'max_physical_memory': {'type': 'integer'},
    'max_swap': {'type': 'integer'},
    'mdata_exec_timeout': {'type': 'integer'},
    'nics': {'type': 'object-array'},
    'nics.*.allow_dhcp_spoofing': {'type': 'boolean'},
    'nics.*.allow_ip_spoofing': {'type': 'boolean'},
    'nics.*.allow_mac_spoofing': {'type': 'boolean'},
    'nics.*.allow_restricted_traffic': {'type': 'boolean'},
    'nics.*.allow_unfiltered_promisc': {'type': 'boolean'},
    'nics.*.blocked_outgoing_ports': {'type': 'list'},
    'nics.*.dhcp_server': {'type': 'boolean'},
    'nics.*.gateway': {'type': 'string'},
    'nics.*.interface': {'type': 'string'},
    'nics.*.ip': {'type': 'string'},
    'nics.*.mac': {'type': 'string'},
    'nics.*.model': {'type': 'string'},
    'nics.*.netmask': {'type': 'string'},
    'nics.*.nic_tag': {'type': 'string'},
    'nics.*.primary': {'type': 'boolean'},
    'nics.*.vrrp_vrid': {'type': 'integer-8bit'},
    'nics.*.vrrp_primary_ip': {'type': 'string'},
    'nics.*.vlan_id': {'type': 'integer'},
    'nic_driver': {'type': 'string'},
    'nowait': {'type': 'boolean'},
    'owner_uuid': {'type': 'string'},
    'package_name': {'type': 'string'},
    'package_version': {'type': 'string'},
    'qemu_opts': {'type': 'string'},
    'qemu_extra_opts': {'type': 'string'},
    'quota': {'type': 'integer'},
    'ram': {'type': 'integer'},
    'remove_customer_metadata': {'type': 'list'},
    'remove_disks': {'type': 'list'},
    'remove_internal_metadata': {'type': 'list'},
    'remove_nics': {'type': 'list'},
    'remove_tags': {'type': 'list'},
    'resolvers': {'type': 'list'},
    'set_tags': {'type': 'flat-object'},
    'set_customer_metadata': {'type': 'flat-object'},
    'set_internal_metadata': {'type': 'flat-object'},
    'spice_opts': {'type': 'string'},
    'spice_password': {'type': 'string'},
    'spice_port': {'type': 'integer'},
    'tags': {'type': 'flat-object'},
    'tmpfs': {'type': 'integer'},
    'transition': {'type': 'flat-object'},
    'update_disks': {'type': 'object-array', 'check_as': 'disks'},
    'update_nics': {'type': 'object-array', 'check_as': 'nics'},
    'uuid': {'type': 'uuid'},
    'vcpus': {'type': 'integer'},
    'vga': {'type': 'string'},
    'virtio_txburst': {'type': 'integer'},
    'virtio_txtimer': {'type': 'integer'},
    'vnc_password': {'type': 'string'},
    'vnc_port': {'type': 'integer'},
    'zfs_data_compression': {'type': 'string'},
    'zfs_data_recsize': {'type': 'integer'},
    'zfs_io_priority': {'type': 'integer'},
    'zfs_root_compression': {'type': 'string'},
    'zfs_root_recsize': {'type': 'integer'},
    'zone_dataset_uuid': {'type': 'uuid'},
    'zonename': {'type': 'string'},
    'zfs_storage_pool_name': {'type': 'zpool'},
    'zpool': {'type': 'zpool'}
};

// shared between 'joyent' and 'joyent-minimal'
var joyent_allowed = {
    'add_nics': ['update'],
    'alias': ['create', 'receive', 'update'],
    'autoboot': ['create', 'receive', 'update'],
    'billing_id': ['create', 'receive', 'update'],
    'brand': ['create', 'receive'],
    'cpu_cap': ['create', 'receive', 'update'],
    'cpu_shares': ['create', 'receive', 'update'],
    'create_only': ['receive'],
    'create_timestamp': ['receive'],
    'customer_metadata': ['create', 'receive'],
    'dataset_uuid': ['create', 'receive'],
    'delegate_dataset': ['create', 'receive'],
    'do_not_inventory': ['create', 'receive', 'update'],
    'dns_domain': ['create', 'receive'],
    'filesystems': ['create', 'receive'],
    'filesystems.*.type': ['add'],
    'filesystems.*.source': ['add'],
    'filesystems.*.target': ['add'],
    'filesystems.*.raw': ['add'],
    'filesystems.*.options': ['add'],
    'fs_allowed': ['create', 'receive', 'update'],
    'hostname': ['create', 'receive', 'update'],
    'image_uuid': ['create', 'receive'],
    'internal_metadata': ['create', 'receive'],
    'limit_priv': ['create', 'receive', 'update'],
    'max_locked_memory': ['create', 'receive', 'update'],
    'max_lwps': ['create', 'receive', 'update'],
    'max_physical_memory': ['create', 'receive', 'update'],
    'max_swap': ['create', 'receive', 'update'],
    'mdata_exec_timeout': ['create'],
    'nics': ['create', 'receive'],
    'nics.*.allow_dhcp_spoofing': ['add', 'update'],
    'nics.*.allow_ip_spoofing': ['add', 'update'],
    'nics.*.allow_mac_spoofing': ['add', 'update'],
    'nics.*.allow_restricted_traffic': ['add', 'update'],
    'nics.*.blocked_outgoing_ports': ['add', 'update'],
    'nics.*.dhcp_server': ['add', 'update'],
    'nics.*.gateway': ['add', 'update'],
    'nics.*.interface': ['add', 'update'],
    'nics.*.ip': ['add', 'update'],
    'nics.*.mac': ['add', 'update'],
    'nics.*.netmask': ['add', 'update'],
    'nics.*.nic_tag': ['add', 'update'],
    'nics.*.vrrp_vrid': ['add', 'update'],
    'nics.*.vrrp_primary_ip': ['add', 'update'],
    'nics.*.primary': ['add', 'update'],
    'nics.*.vlan_id': ['add', 'update'],
    'nowait': ['create', 'receive'],
    'owner_uuid': ['create', 'receive', 'update'],
    'package_name': ['create', 'receive', 'update'],
    'package_version': ['create', 'receive', 'update'],
    'quota': ['create', 'receive', 'update'],
    'ram': ['create', 'receive', 'update'],
    'remove_customer_metadata': ['update'],
    'remove_internal_metadata': ['update'],
    'remove_nics': ['update'],
    'remove_tags': ['update'],
    'resolvers': ['create', 'receive', 'update'],
    'set_customer_metadata': ['update'],
    'set_internal_metadata': ['update'],
    'set_tags': ['update'],
    'tags': ['create', 'receive'],
    'tmpfs': ['create', 'receive', 'update'],
    'transition': ['receive'],
    'update_nics': ['update'],
    'uuid': ['create', 'receive'],
    'zfs_data_compression': ['create', 'receive', 'update'],
    'zfs_data_recsize': ['create', 'receive', 'update'],
    'zfs_io_priority': ['create', 'receive', 'update'],
    'zfs_root_compression': ['create', 'receive', 'update'],
    'zfs_root_recsize': ['create', 'receive', 'update'],
    'zfs_storage_pool_name': ['create', 'receive'],
    'zonename': ['create', 'receive'],
    'zpool': ['create', 'receive']
};

/*
 * This defines all of the properties allowed, required and features that a
 * brand has. For each of the allowed/required properties you have a list of
 * actions for which this is allowed/required. For properties that are lists
 * of objects, you can specify the action as 'add' or 'update' for when you're
 * adding or updating one of those objects.
 *
 * Features can currently be one of:
 *
 * 'check_avail_ram' -- (boolean) whether we should check system ram on create
 * 'cleanup_dataset' -- (boolean) whether to remove trash before booting
 * 'default_memory_overhead' -- (integer) memory above 'ram' that's added
 * 'limit_priv': (list) list of priviledges for this zone (if not 'default')
 * 'min_memory_overhead' -- (integer) minimum delta between ram + max_physical
 * 'model_required' -- (boolean) whether a .model is required on nics and disks
 * 'pid_file' -- (pathname) file containing the PID for zones with one process
 * 'runtime_info' -- (boolean) whether this zone supports the 'info' command
 * 'serial_console' -- (boolean) whether this zone uses serial console
 * 'type' -- the type of the VM (OS or KVM), all brands should include this
 * 'update_mdata_exec_timeout' (boolean) whether to update mdata:exec timeout
 * 'update_rctls' (boolean) whether we can update rctls 'live' for this zone
 * 'use_tmpfs' -- (boolean) whether this type of zone uses tmpfs
 * 'use_vm_autoboot' -- (boolean) use vm-autoboot instead of autoboot
 * 'use_vmadmd' -- (boolean) use vmadmd for some actions instead of direct
 * 'var_svc_provisioning' -- (boolean) whether brand uses /var/svc/provisioning
 * 'wait_for_hwsetup' -- (boolean) use QMP and provision_success when hwsetup
 * 'write_zone_netfiles' -- (boolean) write out files like /etc/hostname.net0
 * 'zlogin_console' -- (boolean) use zlogin -C for console (vs. serial_console)
 * 'zoneinit' -- (boolean) this brand's setup may be controlled by zoneinit
 *
 * All of the keys:
 *
 *  allowed_properties
 *  required_properties
 *  features
 *
 * should be defined for each brand. Even if empty.
 */
var BRAND_OPTIONS = {
    'joyent': {
        'allowed_properties': joyent_allowed,
        'required_properties': {
            'brand': ['create', 'receive'],
            'image_uuid': ['create', 'receive']
        }, 'features': {
            'cleanup_dataset': true,
            'type': 'OS',
            'update_mdata_exec_timeout': true,
            'update_rctls': true,
            'use_tmpfs': true,
            'write_zone_netfiles': true,
            'zlogin_console': true,
            'zoneinit': true
        }
    }, 'joyent-minimal': {
        'allowed_properties': joyent_allowed,
        'required_properties': {
            'brand': ['create', 'receive'],
            'image_uuid': ['create', 'receive']
        }, 'features': {
            'cleanup_dataset': true,
            'type': 'OS',
            'update_mdata_exec_timeout': true,
            'update_rctls': true,
            'use_tmpfs': true,
            'var_svc_provisioning': true,
            'write_zone_netfiles': true,
            'zlogin_console': true
        }
    }, 'kvm': {
        'allowed_properties': {
            'add_disks': ['update'],
            'add_nics': ['update'],
            'alias': ['create', 'receive', 'update'],
            'autoboot': ['create', 'receive', 'update'],
            'billing_id': ['create', 'receive', 'update'],
            'boot': ['create', 'receive', 'update'],
            'brand': ['create', 'receive'],
            'cpu_cap': ['create', 'receive', 'update'],
            'cpu_shares': ['create', 'receive', 'update'],
            'cpu_type': ['create', 'receive', 'update'],
            'create_only': ['receive'],
            'create_timestamp': ['receive'],
            'customer_metadata': ['create', 'receive'],
            'disks': ['create', 'receive'],
            'disks.*.block_size': ['add'],
            'disks.*.boot': ['add', 'update'],
            'disks.*.compression': ['add', 'update'],
            'disks.*.image_name': ['add', 'update'],
            'disks.*.image_size': ['add'],
            'disks.*.image_uuid': ['add'],
            'disks.*.size': ['add'],
            'disks.*.media': ['add', 'update'],
            'disks.*.model': ['add', 'update'],
            'disks.*.nocreate': ['add'],
            'disks.*.path': ['add', 'update'],
            'disks.*.zpool': ['add'],
            'disk_driver': ['create', 'receive', 'update'],
            'do_not_inventory': ['create', 'receive', 'update'],
            'hostname': ['create', 'receive', 'update'],
            'image_uuid': ['create', 'receive'],
            'internal_metadata': ['create', 'receive'],
            'limit_priv': ['create', 'receive', 'update'],
            'max_locked_memory': ['create', 'receive', 'update'],
            'max_lwps': ['create', 'receive', 'update'],
            'max_physical_memory': ['create', 'receive', 'update'],
            'max_swap': ['create', 'receive', 'update'],
            'nics': ['create', 'receive'],
            'nics.*.allow_dhcp_spoofing': ['add', 'update'],
            'nics.*.allow_ip_spoofing': ['add', 'update'],
            'nics.*.allow_mac_spoofing': ['add', 'update'],
            'nics.*.allow_restricted_traffic': ['add', 'update'],
            'nics.*.allow_unfiltered_promisc': ['add', 'update'],
            'nics.*.blocked_outgoing_ports': ['add', 'update'],
            'nics.*.dhcp_server': ['add', 'update'],
            'nics.*.gateway': ['add', 'update'],
            'nics.*.interface': ['add', 'update'],
            'nics.*.ip': ['add', 'update'],
            'nics.*.mac': ['add', 'update'],
            'nics.*.model': ['add'],
            'nics.*.netmask': ['add', 'update'],
            'nics.*.nic_tag': ['add', 'update'],
            'nics.*.primary': ['add', 'update'],
            'nics.*.vlan_id': ['add', 'update'],
            'nic_driver': ['create', 'receive', 'update'],
            'owner_uuid': ['create', 'receive', 'update'],
            'package_name': ['create', 'receive', 'update'],
            'package_version': ['create', 'receive', 'update'],
            'qemu_opts': ['create', 'receive', 'update'],
            'qemu_extra_opts': ['create', 'receive', 'update'],
            'quota': ['create', 'receive', 'update'],
            'ram': ['create', 'receive', 'update'],
            'remove_customer_metadata': ['update'],
            'remove_disks': ['update'],
            'remove_internal_metadata': ['update'],
            'remove_nics': ['update'],
            'remove_tags': ['update'],
            'resolvers': ['create', 'receive', 'update'],
            'set_customer_metadata': ['update'],
            'set_internal_metadata': ['update'],
            'set_tags': ['update'],
            'spice_opts': ['create', 'receive', 'update'],
            'spice_password': ['create', 'receive', 'update'],
            'spice_port': ['create', 'receive', 'update'],
            'transition': ['receive'],
            'update_disks': ['update'],
            'update_nics': ['update'],
            'uuid': ['create', 'receive'],
            'vcpus': ['create', 'receive', 'update'],
            'vga': ['create', 'receive', 'update'],
            'virtio_txburst': ['create', 'receive', 'update'],
            'virtio_txtimer': ['create', 'receive', 'update'],
            'vnc_password': ['create', 'receive', 'update'],
            'vnc_port': ['create', 'receive', 'update'],
            'zfs_io_priority': ['create', 'receive', 'update'],
            'zfs_root_compression': ['create', 'receive', 'update'],
            'zfs_root_recsize': ['create', 'receive', 'update'],
            'zone_dataset_uuid': ['create', 'receive'],
            'zpool': ['create', 'receive']
        }, 'required_properties': {
            'brand': ['create', 'receive']
        }, 'features': {
            'check_avail_ram': true,
            'default_memory_overhead': VM.KVM_MEM_OVERHEAD,
            'limit_priv': ['default', '-file_link_any', '-net_access',
                '-proc_fork', '-proc_info', '-proc_session'],
            'min_memory_overhead': VM.KVM_MIN_MEM_OVERHEAD,
            'model_required': true,
            'pid_file': '/tmp/vm.pid',
            'runtime_info': true,
            'serial_console': true,
            'type': 'KVM',
            'use_vm_autoboot': true,
            'use_vmadmd': true,
            'var_svc_provisioning': true,
            'wait_for_hwsetup': true
        }
    }
};

var VIRTIO_TXTIMER_DEFAULT = 200000;
var VIRTIO_TXBURST_DEFAULT = 128;

function getZpools(callback)
{
    var args = ['list', '-H', '-p', '-o', 'name'];
    var cmd = '/usr/sbin/zpool';
    var idx;
    var raw = [];
    var zpools = [];

    VM.log.debug(cmd + ' ' + args.join(' '));
    execFile(cmd, args, function (error, stdout, stderr) {
        if (error) {
            VM.log.error('Unable to get list of zpools');
            callback(error, {'stdout': stdout, 'stderr': stderr});
        } else {
            // strip out any empty values (last one).
            raw = stdout.split('\n');
            for (idx in raw) {
                if (raw[idx].length > 0) {
                    zpools.push(raw[idx]);
                }
            }
            callback(null, zpools);
        }
    });
}

function validateProperty(brand, prop, value, action, data, errors)
{
    var allowed;
    var k;

    if (!data.hasOwnProperty('zpools')) {
        data.zpools = [];
    }

    if (BRAND_OPTIONS[brand].hasOwnProperty('allowed_properties')) {
        allowed = BRAND_OPTIONS[brand].allowed_properties;
    } else {
        allowed = {};
    }

    if (!errors.hasOwnProperty('bad_values')) {
        errors.bad_values = [];
    }
    if (!errors.hasOwnProperty('bad_properties')) {
        errors.bad_properties = [];
    }

    if (!allowed.hasOwnProperty(prop)) {
        // thie BRAND_OPTIONS doesn't have this property at all
        if (errors.bad_properties.indexOf(prop) === -1) {
            errors.bad_properties.push(prop);
        }
    } else if (!Array.isArray(allowed[prop])
        || allowed[prop].indexOf(action) === -1) {

        // here we've ether got no actions allowed for this value,
        // or just not this one
        if (errors.bad_properties.indexOf(prop) === -1) {
            errors.bad_properties.push(prop);
        }
    }

    if (PAYLOAD_PROPERTIES.hasOwnProperty(prop)) {
        switch (PAYLOAD_PROPERTIES[prop].type) {
        case 'uuid':
            if (typeof (value) === 'string' && !isUUID(value)
                && errors.bad_values.indexOf(prop) === -1) {

                errors.bad_values.push(prop);
            }
            break;
        case 'boolean':
            if (value === 1 || value === '1') {
                VM.log.warn('DEPRECATED: payload uses 1 instead of '
                    + 'true for ' + prop + ', use "true" instead.');
            } else if (typeof (fixBoolean(value)) !== 'boolean'
                && errors.bad_values.indexOf(prop) === -1) {

                errors.bad_values.push(prop);
            }
            break;
        case 'string':
            if (value === undefined || trim(value.toString()) === '') {
                // if set empty/false we'll keep since this is used to unset
                break;
            } else if (typeof (value) !== 'string'
                && errors.bad_values.indexOf(prop) === -1) {

                errors.bad_values.push(prop);
            }
            break;
        case 'integer':
            if (value === undefined || trim(value.toString()) === '') {
                // if set empty/false we'll keep since this is used to unset
                break;
            } else if (((typeof (value) !== 'string'
                && typeof (value) !== 'number')
                || !value.toString().match(/^[0-9]+$/))
                && errors.bad_values.indexOf(prop) === -1) {

                errors.bad_values.push(prop);
            }
            break;
        case 'integer-8bit':
            if (value === undefined || trim(value.toString()) === '') {
                // if set empty/false we'll keep since this is used to unset
                break;
            } else if (((typeof (value) !== 'string'
                && typeof (value) !== 'number')
                || !value.toString().match(/^[0-9]+$/))
                && errors.bad_values.indexOf(prop) === -1
                ) {

                errors.bad_values.push(prop);
                break;
            }
            if (value < 0 || value > 255) {
                errors.bad_values.push(prop);
            }
            break;
        case 'zpool':
            if ((typeof (value) !== 'string'
                || data.zpools.indexOf(value) === -1)
                && errors.bad_values.indexOf(prop) === -1) {

                errors.bad_values.push(prop);
            }
            break;
        case 'flat-object':
            if (typeof (value) !== 'object'
                && errors.bad_values.indexOf(prop) === -1) {

                errors.bad_values.push(prop);
            }
            for (k in value) {
                if (typeof (value[k]) !== 'string'
                    && typeof (value[k]) !== 'number'
                    && typeof (value[k]) !== 'boolean') {

                    if (errors.bad_values.indexOf(prop) === -1) {
                        errors.bad_values.push(prop);
                    }
                    break;
                }
            }
            break;
        case 'list':
            if (typeof (value) === 'string') {
                // really any string could be valid (a one element list)
                break;
            } else if (Array.isArray(value)) {
                for (k in value) {
                    if (typeof (value[k]) !== 'string'
                        && typeof (value[k]) !== 'number') {

                        // TODO: log something more useful here telling them
                        // the type is invalid.
                        if (errors.bad_values.indexOf(prop) === -1) {
                            errors.bad_values.push(prop);
                        }
                        break;
                    }
                    // if this is an array, it can't have commas in the
                    // values. (since we might stringify the list and
                    // we'd end up with something different.
                    if (value[k].toString().indexOf(',') !== -1
                        && errors.bad_values.indexOf(prop) === -1) {

                        errors.bad_values.push(prop);
                    }
                }
            } else {
                // not a valid type
                if (errors.bad_values.indexOf(prop) === -1) {
                    errors.bad_values.push(prop);
                }
            }
            break;
        case 'object-array':
            if (!Array.isArray(value)) {
                if (errors.bad_values.indexOf(prop) === -1) {
                    errors.bad_values.push(prop);
                }
                break;
            }
            for (k in value) {
                if (typeof (value[k]) !== 'object') {
                    if (errors.bad_values.indexOf(prop) === -1) {
                        errors.bad_values.push(prop);
                    }
                    break;
                }
            }
            break;
        default:
            // don't know what type of prop this is, so it's invalid
            if (errors.bad_properties.indexOf(prop) === -1) {
                errors.bad_properties.push(prop);
            }
            break;
        }
    }
}

exports.validate = function (brand, action, payload, callback)
{
    var errors = {
        'bad_values': [],
        'bad_properties': [],
        'missing_properties': []
    };
    var prop;

    ensureLogging('validate', false);

    if (!BRAND_OPTIONS.hasOwnProperty(brand)) {
        callback({'bad_brand': brand});
        return;
    }

    // wrap the whole thing with getZpools so we have the list of pools if we
    // need them.
    getZpools(function (err, zpools) {
        var idx;
        var prefix;
        var required;
        var subprop;
        var subprop_action = '';
        var value;

        if (err) {
            /*
             * this only happens when the zpool command fails which should be
             * very rare, but when it does happen, we continue with an empty
             * zpool list in case they don't need to validate zpools. If they
             * do, every zpool will be invalid which is also what we want since
             * nothing else that uses zpools is likely to work either.
             *
             */
            zpools = [];
        }

        // loop through and weed out ones we don't allow for this action.
        for (prop in payload) {
            validateProperty(brand, prop, payload[prop], action,
                {'zpools': zpools}, errors);

            // special case for complex properties where we want to check
            // foo.*.whatever
            if (PAYLOAD_PROPERTIES.hasOwnProperty(prop)
                && PAYLOAD_PROPERTIES[prop].type === 'object-array'
                && Array.isArray(payload[prop])) {

                if (PAYLOAD_PROPERTIES[prop].hasOwnProperty('check_as')) {
                    prefix = PAYLOAD_PROPERTIES[prop].check_as + '.*.';
                    if (prop.match(/^add_/)) {
                        subprop_action = 'add';
                    } else if (prop.match(/^update_/)) {
                        subprop_action = 'update';
                    }
                } else {
                    // here we've got something like 'disks' which is an add
                    prefix = prop + '.*.';
                    subprop_action = 'add';
                }

                for (idx in payload[prop]) {
                    if (typeof (payload[prop][idx]) === 'object') {
                        // subprop will be something like 'nic_tag'
                        for (subprop in payload[prop][idx]) {
                            value = payload[prop][idx][subprop];
                            validateProperty(brand, prefix + subprop, value,
                                subprop_action, zpools, errors);
                        }
                    } else if (errors.bad_values.indexOf(prop) === -1) {
                        // this is not an object so bad value in the array
                        errors.bad_values.push(prop);
                    }
                }
            }
        }

        if (BRAND_OPTIONS[brand].hasOwnProperty('required_properties')) {
            required = BRAND_OPTIONS[brand].required_properties;
            for (prop in required) {
                if (required[prop].indexOf(action) !== -1
                    && !payload.hasOwnProperty(prop)) {

                    errors.missing_properties.push(prop);
                }
            }
        }

        if (errors.bad_properties.length > 0 || errors.bad_values.length > 0
            || errors.missing_properties.length > 0) {

            callback(errors);
            return;
        }

        callback();
    });
};

function separateCommas(str)
{
    return str.split(',');
}

function unmangleMem(str)
{
    return (Number(str) / (1024 * 1024));
}

function unbase64(str)
{
    return new Buffer(str, 'base64').toString('ascii');
}

function numberify(str)
{
    return Number(str);
}

function startElement(name, attrs, state) {
    var disk = {};
    var key;
    var newobj;
    var nic = {};
    var obj;
    var prop;
    var stack;
    var use;
    var where;

    if (!state.hasOwnProperty('stack')) {
        state.stack = [];
    }
    obj = state.obj;
    stack = state.stack;

    stack.push(name);
    where = stack.join('.');

    if (XML_PROPERTIES.hasOwnProperty(where)) {
        for (key in XML_PROPERTIES[where]) {
            use = XML_PROPERTIES[where][key];
            if (attrs.hasOwnProperty(key)) {
                obj[use] = attrs[key];
            } else if (attrs.hasOwnProperty('name') && attrs.name === key) {
                // attrs use the whacky {name, type, value} stuff.
                obj[use] = attrs['value'];
            }
        }
    } else if (where === 'zone.rctl') {
        stack.push(attrs.name);
    } else if (where === 'zone.network') {
        // new network device
        for (prop in attrs) {
            if (XML_PROPERTIES.nic.hasOwnProperty(prop)) {
                use = XML_PROPERTIES.nic[prop];
                if (prop === 'mac-addr') {
                    // XXX SmartOS inherited the ridiculous MAC formatting from
                    //     Solaris where leading zeros are removed. We should
                    //     Fix that in the OS tools.
                    nic[use] = fixMac(attrs[prop]);
                } else {
                    nic[use] = attrs[prop];
                }
            } else {
                VM.log.debug('unknown net prop: ' + prop);
            }
        }
        if (!obj.hasOwnProperty('networks')) {
            obj.networks = {};
        }
        obj.networks[nic.mac] = nic;
        stack.push(nic.mac);
    } else if (where.match(/zone\.network\...:..:..:..:..:..\.net-attr/)) {
        if (XML_PROPERTIES.nic.hasOwnProperty(attrs.name)) {
            use = XML_PROPERTIES.nic[attrs.name];
            obj.networks[stack[2]][use] = attrs.value;
        } else {
            VM.log.debug('unknown net prop: ' + attrs.name);
        }
    } else if (where === 'zone.device') {
        // new disk device
        for (prop in attrs) {
            if (XML_PROPERTIES.disk.hasOwnProperty(prop)) {
                use = XML_PROPERTIES.disk[prop];
                disk[use] = attrs[prop];
            } else {
                VM.log.debug('unknown disk prop: ' + prop);
            }
        }
        if (!obj.hasOwnProperty('devices')) {
            obj.devices = {};
        }
        obj.devices[disk.path] = disk;
        stack.push(disk.path);
    } else if (where.match(/zone\.device\.\/.*\.net-attr/)) {
        if (XML_PROPERTIES.disk.hasOwnProperty(attrs.name)) {
            use = XML_PROPERTIES.disk[attrs.name];
            obj.devices[stack[2]][use] = attrs.value;
        } else {
            VM.log.debug('unknown disk prop: ' + attrs.name);
        }
    } else if (where === 'zone.dataset') {
        if (!obj.hasOwnProperty('datasets')) {
            obj.datasets = [];
        }
        if (attrs.hasOwnProperty('name')) {
            obj.datasets.push(attrs.name);
        }
    } else if (where === 'zone.filesystem') {
        if (!obj.hasOwnProperty('filesystems')) {
            obj.filesystems = [];
        }
        newobj = {};
        for (prop in XML_PROPERTIES.filesystem) {
            if (attrs.hasOwnProperty(prop)) {
                newobj[XML_PROPERTIES.filesystem[prop]] = attrs[prop];
            }
        }
        obj.filesystems.push(newobj);
    } else if (where === 'zone.filesystem.fsoption') {
        newobj = obj.filesystems.slice(-1)[0];  // the last element
        if (!newobj.hasOwnProperty('options')) {
            newobj.options = [];
        }
        newobj.options.push(attrs.name);
    } else {
        VM.log.debug('unknown property: ' + where + ': '
            + JSON.stringify(attrs));
    }
}

function endElement(name, state) {
    // trim stack back above this element
    var stack = state.stack;

    while (stack.pop() !== name) {
        // do nothing, we just want to consume.
        continue;
    }
}

function indexSort(obj, field, pattern)
{
    obj.sort(function (a, b) {
        var avalue = 0;
        var bvalue = 0;
        var matches;

        if (a.hasOwnProperty(field)) {
            matches = a[field].match(pattern);
            if (matches) {
                avalue = Number(matches[1]);
            }
        }
        if (b.hasOwnProperty(field)) {
            matches = b[field].match(pattern);
            if (matches) {
                bvalue = Number(matches[1]);
            }
        }

        return avalue - bvalue;
    });
}

function applyTransforms(obj)
{
    var p;
    var pp;
    var subobj;
    var transforms = XML_PROPERTY_TRANSFORMS;

    for (p in transforms) {
        if (obj.hasOwnProperty(p)) {
            if (typeof (transforms[p]) === 'object') {
                // this is a 'complex' property like nic, and has different
                // transforms for the sub-objects
                for (pp in transforms[p]) {
                    for (subobj in obj[p]) {
                        if (obj[p][subobj].hasOwnProperty(pp)) {
                            obj[p][subobj][pp] =
                                transforms[p][pp](obj[p][subobj][pp]);
                        }
                    }
                }
            } else { // function
                obj[p] = transforms[p](obj[p]);
            }
        }
    }
}

function getVmobj(zonename, preload_data, callback)
{
    var filename = '/etc/zones/' + zonename + '.xml';
    var parser = new expat.Parser('UTF-8');

    fs.readFile(filename, function (error, data) {
        var allowed;
        var disk;
        var fields;
        var nic;
        var obj = {};
        var state = {};

        if (error) {
            callback(error);
            return;
        }

        state.obj = obj;
        parser.on('startElement', function (name, attrs) {
            return startElement(name, attrs, state);
        });
        parser.on('endElement', function (name) {
            return endElement(name, state);
        });

        if (!parser.parse(data.toString())) {
            throw new Error('There are errors in your xml file: '
                + parser.getError());
        }

        // now that we know which brand we are, find out what we're allowed.
        allowed = BRAND_OPTIONS[obj.brand].allowed_properties;

        // replace obj.networks with array of nics.
        obj.nics = [];
        for (nic in obj.networks) {
            obj.nics.push(obj.networks[nic]);
        }
        delete obj.networks;

        // replace obj.devices with array of disks.
        if (allowed.hasOwnProperty('disks')) {
            obj.disks = [];
            for (disk in obj.devices) {
                obj.disks.push(obj.devices[disk]);
            }
        }
        delete obj.devices;

        if (!BRAND_OPTIONS.hasOwnProperty(obj.brand)) {
            throw new Error('unable to handle brand ' + obj.brand);
        }

        if (BRAND_OPTIONS[obj.brand].features.use_vm_autoboot) {
            obj.autoboot = obj.vm_autoboot;
            delete obj.vm_autoboot;
        }

        // apply the XML_PROPERTY_TRANSFORMs
        applyTransforms(obj);

        // probe for some fields on disks if this brand of zone supports them.
        if (allowed.hasOwnProperty('disks')
            && (allowed.disks.indexOf('create') !== -1)) {

            for (disk in obj.disks) {
                disk = obj.disks[disk];

                if (preload_data.hasOwnProperty('dsinfo')
                    && preload_data.dsinfo.hasOwnProperty(disk.path)) {

                    disk.zfs_filesystem =
                        preload_data.dsinfo[disk.path].dataset;
                    disk.zpool = disk.zfs_filesystem.split('/')[0];
                }
            }
        }

        if (obj.hasOwnProperty('transition')) {
            fields = rtrim(obj.transition).split(':');
            if (fields.length === 3) {
                delete obj.transition;
                obj.state = fields[0];
                obj.transition_to = fields[1];
                obj.transition_expire = fields[2];
            } else {
                VM.log.debug('getVmobj() ignoring bad value for '
                    + 'transition "' + obj.transition + '"');
            }
        }

        // sort the disks + nics by index
        if (obj.hasOwnProperty('disks')) {
            indexSort(obj.disks, 'path', /^.*-disk(\d+)$/);
        }
        if (obj.hasOwnProperty('nics')) {
            indexSort(obj.nics, 'interface', /^net(\d+)$/);
        }
        if (obj.hasOwnProperty('filesystems')) {
            indexSort(obj.filesystems, 'target', /^(.*)$/);
        }

        callback(null, obj);
    });
}

function setQuota(dataset, quota, callback)
{
    var newval;

    if (!dataset) {
        callback(new Error('Invalid dataset: "' + dataset + '"'));
        return;
    }

    if (quota === 0 || quota === '0') {
        newval = 'none';
    } else {
        newval = quota.toString() + 'g';
    }

    zfs(['set', 'quota=' + newval, dataset], function (err, fds) {
        if (err) {
            VM.log.error('setQuota() cmd failed: ' + fds.stderr);
            callback(new Error(rtrim(fds.stderr)));
        } else {
            callback();
        }
    });
}

/*
 * This can either get a list of all datasets and their quotas + mountpoint, or
 * the info for a specific dataset if a vm object is passed that has a
 * 'zonepath'.
 */
function loadDatasetInfo(m, callback)
{
    var args;

    args = ['list', '-H', '-p', '-t', 'volume,filesystem,snapshot', '-o',
        'name,quota,volsize,mountpoint,type,compression,recsize,'
        + 'volblocksize,zoned'];

    if (m && m.hasOwnProperty('zonepath')) {
        args.push(m.zonepath);
    }

    // no VM, so we want all VM's quotas
    zfs(args, function (err, fds) {
        var compression;
        var dataset;
        var dsinfo = {};
        var fields;
        var infokey;
        var line;
        var lines = fds.stdout.split('\n');
        var mountpoint;
        var quota;
        var recsize;
        var snapname;
        var snapparts;
        var type;
        var volblocksize;
        var volsize;
        var zoned;

        for (line in lines) {
            line = trim(lines[line]);
            fields = line.split(/\s+/);

            if (fields.length === 9) {
                dataset = fields[0];
                quota = fields[1];
                if (quota !== '-') {
                    quota = Number(quota) / (1024 * 1024 * 1024); // GiB
                }
                volsize = fields[2];
                if (volsize !== '-') {
                    volsize = Number(volsize) / (1024 * 1024); // MiB
                }
                mountpoint = fields[3];
                type = fields[4];
                compression = fields[5];
                recsize = fields[6];
                if (recsize !== '-') {
                    recsize = Number(recsize);
                }
                volblocksize = fields[7];
                if (volblocksize !== '-') {
                    volblocksize = Number(volblocksize);
                }
                zoned = fields[8];
                if (zoned === 'on') {
                    zoned = true;
                } else {
                    zoned = false;
                }

                if (type === 'volume') {
                    mountpoint = '/dev/zvol/rdsk/' + dataset;
                } else if (type == 'snapshot') {
                    // The only thing we do for snapshots is store the snapshot
                    // name keyed on the dataset name.  So we can include the
                    // list of snapshots for a dataset on the VM.
                    snapparts = dataset.split('@');
                    assert.equal(snapparts.length, 2);
                    snapname = dataset;
                    dataset = snapparts[0];
                    if (!dsinfo.hasOwnProperty('snapshots')) {
                        dsinfo.snapshots = {};
                    }
                    if (!dsinfo.snapshots.hasOwnProperty(dataset)) {
                        dsinfo.snapshots[dataset] = [];
                    }
                    dsinfo.snapshots[dataset].push({'name': snapname});
                    continue;
                } else if (mountpoint === '-' || mountpoint === 'legacy') {
                    mountpoint = '/' + dataset;
                }

                /*
                 *
                 * For non-zoned filesystem datasets (these should be the
                 * zoneroot datasets), we use mountpoint as the key as zonepath
                 * comes from zoneadm's "cheap" info so we can match them up.
                 *
                 * For volumes (KVM VM's disks) we also use mountpoint as we've
                 * set that to the block device path and that's available from
                 * the devices section of the zoneconfig.
                 *
                 * For zoned filesystems (delegated datasets) we use the
                 * dataset name as the mountpoint can be changed from within the
                 * zone.  When a delegated dataset is assigned but the zone's
                 * not been booted, the dataset will not have the 'zoned'
                 * property.  So we also check if the name ends in /data.
                 *
                 */
                infokey = mountpoint;
                if (zoned || (dataset.split('/')[2] === 'data')) {
                    // delegated
                    infokey = dataset;
                }
                dsinfo[infokey] = {
                    'quota': quota,
                    'volsize': volsize,
                    'dataset': dataset,
                    'compression': compression
                };

                if (type === 'volume' && volblocksize !== '-') {
                    dsinfo[infokey].volblocksize = volblocksize;
                } else if (recsize !== '-') {
                    dsinfo[infokey].recsize = recsize;
                }
            } else if (trim(line).length > 0) {
                VM.log.debug('loadDatasetQuota(): ignoring line: ' + line);
            }
        }

        callback(null, dsinfo);
    });
}

function loadTags(vmcfg, callback)
{
    var filename;

    if (vmcfg.zonepath) {
        filename = vmcfg.zonepath + '/config/tags.json';
        VM.log.debug('loadTags() loading tags from ' + filename);

        fs.readFile(filename, function (error, data) {
            var tags;

            if (error) {
                if (error.code === 'ENOENT') {
                    VM.log.debug('Skipping nonexistent file ' + filename);
                    tags = {};
                } else {
                    VM.log.error('loadTags() failed to load tags'
                        + ' from ' + filename, error);
                    callback(error);
                    return;
                }
            } else {
                try {
                    tags = JSON.parse(data.toString());
                } catch (e) {
                    tags = {};
                }
            }

            callback(null, tags);
        });
    } else {
        callback(null, {});
    }
}

function loadMetadata(vmcfg, callback)
{
    var filename;

    if (vmcfg.zonepath) {
        filename = vmcfg.zonepath + '/config/metadata.json';
        VM.log.debug('loadMetadata() loading metadata from ' + filename);

        fs.readFile(filename, function (error, data) {
            var metadata;

            if (error) {
                if (error.code === 'ENOENT') {
                    VM.log.debug('Skipping nonexistent file ' + filename);
                    metadata = {};
                } else {
                    callback(error);
                }
            } else {
                try {
                    metadata = JSON.parse(data.toString());
                } catch (e) {
                    metadata = {};
                }
            }

            loadTags(vmcfg, function (err, tags) {
                if (err) {
                    callback(null, metadata, {});
                } else {
                    callback(null, metadata, tags);
                }
            });
        });
    } else {
        callback(null, {}, {});
    }
}

/*
 * This preloads some data for us that comes from commands which output for
 * *all* VMs.  This allows us to just run these (expensive) commands once
 * instead of having to run them for each VM.
 *
 */
function preloadZoneData(uuid, callback)
{
    var data = {};

    // NOTE: uuid can be null, in which case we get data for all VMs.

    async.series([
        function (cb) {
            getZoneRecords(uuid, function (err, records) {
                if (!err) {
                    data.records = records;
                }
                cb(err);
            });
        }, function (cb) {
            loadDatasetInfo(null, function (err, dsinfo) {
                if (!err) {
                    data.dsinfo = dsinfo;
                }
                cb(err);
            });
        }, function (cb) {
            VM.getSysinfo(function (err, sysinfo) {
                if (!err) {
                    data.sysinfo = sysinfo;
                }
                cb(err);
            });
        }, function (cb) {
            var u;
            var uuids = [];

            for (u in data.records) {
                uuids.push(u);
            }
            async.forEachSeries(uuids, function (z_uuid, zcb) {
                var filename;
                var z = data.records[z_uuid];

                // NOTE: z.state here is equivalent to zone_state not state.
                if (z && BRAND_OPTIONS[z.brand].hasOwnProperty('features')
                    && BRAND_OPTIONS[z.brand].features.pid_file
                    && z.state === 'running') {

                    filename = path.join(z.zonepath, 'root',
                        BRAND_OPTIONS[z.brand].features.pid_file);
                    VM.log.debug('checking for ' + filename);

                    fs.readFile(filename,
                        function (error, filedata) {

                        var pid;

                        if (!error) {
                            pid = Number(trim(filedata.toString()));
                            if (pid > 0) {
                                z.pid = pid;
                                VM.log.debug('found PID ' + pid + ' for '
                                    + z.uuid);
                            }
                        } else {
                            if (error.code === 'ENOENT') {
                                VM.log.debug('no PID file for ' + z.uuid);
                                zcb();
                            } else {
                                zcb(error);
                                return;
                            }
                        }
                    });
                }
                zcb();
            }, function (err) {
                cb(err);
            });
        }
    ], function (err, res) {
        VM.log.trace('leaving preloadZoneData()');
        callback(err, data);
    });
}

function getZoneRecords(uuid, callback)
{
    var args = [];
    var results = {};

    if (uuid) {
        // this gives us zone info if uuid is *either* a zonename or uuid
        if (isUUID(uuid)) {
            args.push('-z');
            args.push(uuid);
            args.push('-u');
            args.push(uuid);
        } else {
            args.push('-z');
            args.push(uuid);
        }
    }
    args.push('list');
    args.push('-p');
    if (!uuid) {
        args.push('-c');
    }

    zoneadm(args, function (err, fds) {
        var errmsg;
        var fields;
        var line;
        var lines;
        var obj;

        if (err) {
            errmsg = rtrim(fds.stderr);
            if (!errmsg.match(/No such zone configured$/)) {
                // not existing isn't always a problem (eg. existence check), so
                // don't always log it as an error.
                VM.log.error(err, 'getZoneRecords() zoneadm "'
                    + args.join(',') + '" failed');
            }

            callback(new Error(errmsg));
            return;
        }

        lines = fds.stdout.split('\n');
        for (line in lines) {
            fields = rtrim(lines[line]).split(':');
            if (fields.length === 8 && fields[1] !== 'global') {
                obj = {
                    'zoneid': Number(fields[0]),
                    'zonename': fields[1],
                    'state': fields[2],
                    'zonepath': fields[3],
                    'uuid': fields[4],
                    'brand': fields[5],
                    'ip_type': fields[6]
                };
                VM.log.debug('loaded: ' + JSON.stringify(obj));
                // XXX zones in some states have no uuid. We should either fix
                //     that or use zonename for those.
                results[obj.uuid] = obj;
            } else if (lines[line].replace(/ /g, '').length > 0) {
                VM.log.debug('getZoneRecords(' + uuid + ') ignoring line: '
                    + lines[line]);
            }
        }
        callback(null, results);
    });
}

exports.flatten = function (vmobj, key)
{
    var tokens = key.split('.');

    ensureLogging('flatten', false);

    if (tokens.length === 3
        && VM.FLATTENABLE_ARRAY_HASH_KEYS.indexOf(tokens[0]) !== -1) {

        if (!vmobj.hasOwnProperty(tokens[0])) {
            return undefined;
        }
        if (!vmobj[tokens[0]].hasOwnProperty(tokens[1])) {
            return undefined;
        }
        return vmobj[tokens[0]][tokens[1]][tokens[2]];
    }

    if (tokens.length === 2
        && VM.FLATTENABLE_HASH_KEYS.indexOf(tokens[0]) !== -1) {

        if (!vmobj.hasOwnProperty(tokens[0])) {
            return undefined;
        }
        return vmobj[tokens[0]][tokens[1]];
    }

    return vmobj[key];
};

function getLastModified(vmobj)
{
    var files = [];
    var file;
    var stat;
    var timestamp = new Date(0);

    if (vmobj.zonepath) {
        files.push(path.join(vmobj.zonepath, '/config/metadata.json'));
        files.push(path.join(vmobj.zonepath, '/config/tags.json'));
    } else {
        VM.log.debug('getLastModified() no zonepath!');
    }

    if (vmobj.hasOwnProperty('zonename')) {
        files.push('/etc/zones/' + vmobj.zonename + '.xml');
    } else {
        VM.log.debug('getLastModified() no zonename!');
    }

    for (file in files) {
        file = files[file];
        try {
            stat = fs.statSync(file);
            if (stat.isFile()) {
                if (stat.mtime > timestamp) {
                    timestamp = stat.mtime;
                }
            }
        } catch (e) {
            if (e.code !== 'ENOENT') {
                VM.log.error(e, 'Unable to get timestamp for "' + file + '":'
                    + e.message);
            }
        }
    }

    return (timestamp.toISOString());
}

function loadVM(uuid, data, callback)
{
    var e;
    var info;

    // XXX need to always have data when we get here
    info = data.records[uuid];

    if (!info) {
        e = new Error('VM.load() empty info when getting record '
            + 'for vm ' + uuid);
        VM.log.error(e);
        callback(e);
        return;
    }

    getVmobj(info.zonename, data, function (err, vmobj) {
        if (err) {
            callback(err);
            return;
        }

        // add the bits we can only find from zoneadm list
        vmobj.uuid = info.uuid;

        // find when we last modified this VM
        vmobj.last_modified = getLastModified(vmobj);

        // In the case of 'configured' zones, we might only have zonename
        // because uuid isn't set yet.  Because of that case, we set uuid
        // to zonename if it is in UUID form.
        if ((!vmobj.uuid || vmobj.uuid.length === 0)
            && isUUID(vmobj.zonename)) {

            vmobj.uuid = vmobj.zonename;
        }

        // sysinfo has server_uuid and potentially some DC info
        if (data.hasOwnProperty('sysinfo')) {
            if (data.sysinfo.hasOwnProperty('UUID')) {
                vmobj.server_uuid = data.sysinfo.UUID;
            }
            if (data.sysinfo.hasOwnProperty('Datacenter Name')) {
                vmobj.datacenter_name = data.sysinfo['Datacenter Name'];
            }
            if (data.sysinfo.hasOwnProperty('Headnode ID')) {
                vmobj.headnode_id = data.sysinfo['Headnode ID'];
            }
        }

        if (info.zoneid !== '-') {
            vmobj.zoneid = info.zoneid;
        }

        if (info.pid) {
            vmobj.pid = info.pid;
        }

        // state could already be set here if it was overriden by a transition
        // that's in progress. So we only change if that's not the case.
        vmobj.zone_state = info.state;
        if (!vmobj.hasOwnProperty('state')) {
            if (info.state === 'installed') {
                vmobj.state = 'stopped';
            } else {
                vmobj.state = info.state;
            }
        }

        // If the zone has the 'failed' property it doesn't matter what
        // other state it might be in, we list its state as 'failed'.
        if (vmobj.failed) {
            vmobj.state = 'failed';
        }

        loadMetadata(vmobj, function (error, metadata, tags) {
            var d;
            var delegated;
            var disk;
            var disk_info;
            var ds;
            var dsinfo;
            var filesys;
            var k;
            var snapshots = [];

            // when zone_state is 'incomplete' we could be deleting it in which
            // case metadata may already be gone, ignore failure to load mdata
            // when 'incomplete' because of this.
            if (error) {
                if (vmobj.zone_state !== 'incomplete') {
                    VM.log.debug(error, 'zone is in state incomplete ignoring '
                        + 'error: ' + error.message);
                } else {
                    callback(new Error('Unable to add metadata:'
                        + error.message));
                    return;
                }
            }

            // local alias
            dsinfo = data.dsinfo;

            if (dsinfo.hasOwnProperty(vmobj.zonepath)) {
                vmobj.quota = dsinfo[vmobj.zonepath].quota;
                VM.log.debug('found quota "' + vmobj.quota + '" for '
                    + vmobj.uuid);
                if (dsinfo[vmobj.zonepath].hasOwnProperty('compression')
                    && (dsinfo[vmobj.zonepath].compression !== 'off')) {

                    vmobj.zfs_root_compression =
                        dsinfo[vmobj.zonepath].compression;
                }
                if (dsinfo[vmobj.zonepath].hasOwnProperty('recsize')) {
                    vmobj.zfs_root_recsize =
                        dsinfo[vmobj.zonepath].recsize;
                }
                vmobj.zfs_filesystem = dsinfo[vmobj.zonepath].dataset;

                // If there are snapshots for this dataset, add them to the list
                if (dsinfo.hasOwnProperty('snapshots')
                    && dsinfo.snapshots.hasOwnProperty(vmobj.zfs_filesystem)) {

                    snapshots = snapshots.concat(
                        dsinfo.snapshots[vmobj.zfs_filesystem]);
                }

                VM.log.debug('found dataset "' + vmobj.zfs_filesystem
                    + '" for ' + vmobj.uuid);
            } else {
                VM.log.debug('no dsinfo for ' + vmobj.uuid);
            }

            // delegated datasets are keyed on the dataset name instead of
            // mountpoint, since mountpoint can change in a zone.
            if (vmobj.hasOwnProperty('zfs_filesystem')) {
                delegated = vmobj.zfs_filesystem + '/data';
                if (dsinfo.hasOwnProperty(delegated)) {
                    if (dsinfo[delegated].hasOwnProperty('compression')
                        && (dsinfo[delegated].compression !== 'off')) {

                        vmobj.zfs_data_compression =
                            dsinfo[delegated].compression;
                    }
                    if (dsinfo[delegated].hasOwnProperty('recsize')) {
                        vmobj.zfs_data_recsize = dsinfo[delegated].recsize;
                    }

                    // If there are snapshots for this dataset, add them
                    if (dsinfo.hasOwnProperty('snapshots')
                        && dsinfo.snapshots.hasOwnProperty(delegated)) {

                        snapshots =
                            snapshots.concat(dsinfo.snapshots[delegated]);
                    }
                } else {
                    VM.log.debug('no dsinfo for ' + delegated);
                }

                vmobj.zpool =
                    vmobj.zfs_filesystem.split('/')[0];
            }

            if (vmobj.hasOwnProperty('disks')) {
                for (d in vmobj.disks) {
                    d = vmobj.disks[d];
                    if (d.hasOwnProperty('path')
                        && dsinfo.hasOwnProperty(d.path)) {

                        disk_info = dsinfo[d.path];
                        if (disk_info.hasOwnProperty('volsize')) {
                            d.size = disk_info.volsize;
                            VM.log.debug('found size=' + d.size + ' for '
                                + JSON.stringify(d));
                        }
                        if (disk_info.hasOwnProperty('compression')) {
                            d.compression = disk_info.compression;
                        }
                        if (disk_info.hasOwnProperty('volblocksize')) {
                            d.block_size = disk_info.volblocksize;
                        }

                        // If there are snapshots for this dataset, add them to
                        // the list.
                        if (dsinfo.hasOwnProperty('snapshots')
                            && dsinfo.snapshots.hasOwnProperty(
                                d.zfs_filesystem)) {

                            snapshots = snapshots.concat(snapshots,
                                dsinfo.snapshots[d.zfs_filesystem]);
                        }
                    } else if (d.hasOwnProperty('path')) {
                        d.missing = true;
                    } else {
                        VM.log.warn('no dsinfo and no path for '
                            + JSON.stringify(d));
                    }
                }
            }

            for (k in metadata) {
                vmobj[k] = metadata[k];
            }
            if (!vmobj.hasOwnProperty('customer_metadata')) {
                vmobj.customer_metadata = {};
            }
            if (!vmobj.hasOwnProperty('internal_metadata')) {
                vmobj.internal_metadata = {};
            }
            if (tags) {
                vmobj.tags = tags;
            } else {
                vmobj.tags = {};
            }

            vmobj.snapshots = snapshots;

            if (vmobj.state === 'receiving') {
                vmobj.missing = { 'datasets': [], 'disks': [],
                    'filesystems': [] };
                if (!fs.existsSync(vmobj.zonepath)) {
                    vmobj.missing.datasets.push(vmobj.zonepath.substr(1));
                }
                for (ds in vmobj.datasets) {
                    ds = vmobj.datasets[ds];
                    vmobj.missing.datasets.push(ds);
                }
                for (filesys in vmobj.filesystems) {
                    filesys = vmobj.filesystems[filesys];
                    if (filesys.hasOwnProperty('source')) {
                        vmobj.missing.filesystems.push(filesys.source);
                    }
                }
                for (disk in vmobj.disks) {
                    disk = vmobj.disks[disk];
                    if (disk.hasOwnProperty('missing')) {
                        vmobj.missing.disks.push(disk.path);
                    }
                }
            }

            callback(null, vmobj);
        });
    });
}

exports.load = function (uuid, callback)
{
    // This is a wrapper so that other internal functions here (such as lookup)
    // can do smart things like check the quota for each VM with a separate call
    // to zfs get.

    ensureLogging('load', false);

    preloadZoneData(uuid, function (error, data) {
        if (error) {
            VM.log.error(error, 'VM.load() failed to get zone record'
                + ' for ' + uuid);
            callback(error);
        } else {
            loadVM(uuid, data, callback);
        }
    });
};

function fixMac(str)
{
    var fixed = [];
    var octet;
    var octets = str.split(':');

    for (octet in octets) {
        if (octets.hasOwnProperty(octet)) {
            octet = parseInt(octets[octet], 16);
            if (octet === 'nan') {
                octet = 0;
            }
            fixed.push(sprintf('%02x', octet));
        }
    }

    return fixed.join(':');
}

// zonecfg requires removing leading 0's in MACs like 01:02:03:04:05:06
// This function takes a MAC in normal form and puts it in the goofy form
// zonecfg wants.
function ruinMac(mac)
{
    var part;
    var parts;
    var out = [];

    parts = mac.split(':');

    for (part in parts) {
        part = ltrim(parts[part], '0');
        if (part.length === 0) {
            part = '0';
        }
        out.push(part);
    }

    return (out.join(':'));
}

function matcher(zone, search)
{
    var fields;
    var found;
    var i;
    var key;
    var parameters_matched = 0;
    var regex;
    var target;

    function find_match(k, targ) {
        var value = VM.flatten(zone, k);

        if (!regex && k.match(/^nics\..*\.mac$/)) {
            // Fix for broken SmartOS MAC format
            targ = fixMac(targ);
        }

        if (regex && value && value.toString().match(targ)) {
            found = true;
        } else if (value && value.toString() === targ.toString()) {
            found = true;
        }
    }

    for (key in search) {
        found = false;
        regex = false;

        target = search[key];
        if (target[0] === '~') {
            regex = true;
            target = new RegExp(target.substr(1), 'i');
        }

        fields = key.split('.');
        if (fields.length === 3 && fields[1] === '*'
            && VM.FLATTENABLE_ARRAY_HASH_KEYS.indexOf(fields[0]) !== -1) {

            // Special case: for eg. nics.*.ip, we want to loop through all nics
            for (i = 0; i < zone[fields[0]].length; i++) {
                fields[1] = i;
                find_match(fields.join('.'), target);
            }
        } else {
            find_match(key, target);
        }

        if (!found) {
            return false;
        } else {
            parameters_matched++;
        }
    }

    if (parameters_matched > 0) {
        // we would have returned false from the loop had any parameters not
        // matched and we had at least one that did.
        return true;
    }

    return false;
}

exports.lookup = function (search, options, callback) {
    var key;
    var quick_ok = true;
    var results = [];
    var transform;

    ensureLogging('lookup', false);

    if (!options) {
        options = {};
    }

    // XXX the 'transform' option is not intended to be public yet and should
    // only be used by tools willing to be rewritten if this is removed or
    // changed.
    if (options.hasOwnProperty('transform')) {
        transform = options.transform;
    }

    for (key in search) {
        if (QUICK_LOOKUP.indexOf(key) === -1) {
            quick_ok = false;
        }
    }
    preloadZoneData(null, function (err, data) {
        var records = data.records;
        var uuids = [];

        if (err) {
            callback(err);
            return;
        }

        if (quick_ok) {
            var full_results = [];
            var match;
            var regex;
            var source;
            var target;
            var u;
            var z;

            if (err) {
                callback(err);
                return;
            }
            for (z in records) {
                z = records[z];
                match = true;
                for (key in search) {
                    regex = false;
                    // force field type to string so that earlier transformed
                    // number fields get back their match method and the
                    // strict not-equal operator will work on number lookups
                    source = '' + z[key];
                    target = search[key];
                    if (target[0] === '~') {
                        target = new RegExp(target.substr(1), 'i');
                        regex = true;
                    }
                    if (regex && !source.match(target)) {
                        match = false;
                    } else if (!regex && (source !== search[key])) {
                        match = false;
                    }
                }
                if (match && z.uuid) {
                    results.push(z.uuid);
                }
            }
            if (!options.full) {
                // we don't need all the data so what we already got is enough
                callback(null, results);
                return;
            } else {
                // need to expand these
                function expander(uuid, cb)
                {
                    loadVM(uuid, data, function (e, obj) {
                        if (e) {
                            if (e.code === 'ENOENT') {
                                // zone likely was deleted since lookup, ignore
                                cb();
                            } else {
                                cb(e);
                            }
                        } else {
                            if (transform) {
                                transform(obj);
                            }
                            full_results.push(obj);
                            cb();
                        }
                    });
                }

                // create all the volumes we found that we need.
                async.forEachSeries(results, expander, function (e) {
                    if (e) {
                        VM.log.error(e, 'VM.lookup failed to expand results: '
                            + e.message);
                        callback(e);
                    } else {
                        callback(null, full_results);
                    }
                });
            }
        } else {
            // have to search the hard way (through all the data)
            for (u in records) {
                uuids.push(u);
            }
            // this is parallel!
            async.forEach(uuids, function (uuid, cb) {
                var vmobj = records[uuid];

                loadVM(vmobj.uuid, data, function (error, obj) {
                    if (error) {
                        if (error.code === 'ENOENT') {
                            // zone likely was deleted since lookup, ignore
                            cb();
                        } else {
                            cb(error);
                        }
                    } else {
                        if (transform) {
                            transform(obj);
                        }
                        if (matcher(obj, search)) {
                            results.push(obj);
                        }
                        cb();
                    }
                });
            }, function (e) {
                var r;
                var short_results = [];

                if (e) {
                    callback(e);
                } else {
                    if (options.full) {
                        callback(null, results);
                    } else {
                        for (r in results) {
                            short_results.push(results[r].uuid);
                        }
                        callback(null, short_results);
                    }
                }
            });
        }
    });
};

// create a random new locally administered MAC address
function generateMAC()
{
    var data = [(Math.floor(Math.random() * 15) + 1).toString(16) + 2];
    for (var i = 0; i < 5; i++) {
        var oct = (Math.floor(Math.random() * 255) + 1).toString(16);
        if (oct.length == 1) {
            oct = '0' + oct;
        }
        data.push(oct);
    }

    return data.join(':');
}

// return the MAC address based on a VRRP Virtual Router ID
function vrrpMAC(vrid) {
    return sprintf('00:00:5e:00:01:%02x', vrid);
}

// Ensure we've got all the datasets necessary to create this VM
//
// IMPORTANT:
//
// On SmartOS, we assume a provisioner or some other external entity has already
// loaded the dataset into the system. This function just confirms that the
// dataset actually exists.
//
function checkDatasets(payload, callback)
{
    var checkme = [];
    var d;
    var disk;

    VM.log.debug('Checking for required datasets.');

    // build list of datasets we need to download (downloadme)
    for (disk in payload.add_disks) {
        if (payload.add_disks.hasOwnProperty(disk)) {
            d = payload.add_disks[disk];
            if (d.hasOwnProperty('image_uuid')) {
                checkme.push(payload.zpool + '/'
                    + d.image_uuid);
            }
        }
    }

    function checker(dataset, cb) {
        zfs(['list', '-o', 'name', '-H', dataset], function (err, fds) {
            if (err) {
                VM.log.error({'err': err, 'stdout': fds.stdout,
                    'stderr': fds.stderr}, 'zfs list ' + dataset + ' '
                    + 'exited with' + ' code ' + err.code + ': ' + err.message);
                cb(new Error('unable to find dataset: ' + dataset));
            } else {
                cb();
            }
        });
    }

    // check that we have all the volumes
    async.forEachSeries(checkme, checker, function (err) {
        if (err) {
            VM.log.error(err, 'checkDatasets() failed to find required '
                + 'volumes');
            callback(err);
        } else {
            // progress(100, 'we have all necessary datasets');
            callback();
        }
    });
}

function lookupConflicts(macs, ips, vrids, callback) {
    var conflict = false;

    VM.log.debug('checking for conflicts with '
        + JSON.stringify(macs) + ', ' + JSON.stringify(ips) + ' and '
        + JSON.stringify(vrids));

    if (macs.length === 0 && ips.length === 0 && vrids.length === 0) {
        VM.log.debug('returning from conflict check (nothing to check)');
        callback(null, conflict);
        return;
    }

    preloadZoneData(null, function (err, data) {
        var records = data.records;
        var uuid;
        var uuids = [];

        if (err) {
            callback(err);
            return;
        }

        for (uuid in records) {
            uuids.push(uuid);
        }

        // this is parallel!
        async.forEach(uuids, function (z_uuid, cb) {
            loadVM(z_uuid, data, function (error, obj) {
                var ip;
                var mac;
                var vrid;

                if (error) {
                    if (error.code === 'ENOENT') {
                        // zone likely was deleted since lookup, ignore it
                        cb();
                    } else {
                        cb(error);
                    }
                    return;
                }
                for (ip in ips) {
                    if (ips[ip] !== 'dhcp'
                        && matcher(obj, {'nics.*.ip': ips[ip]})) {

                        VM.log.error('Found conflict: ' + obj.uuid
                            + ' already has IP ' + ips[ip]);
                        conflict = true;
                    }
                }
                for (mac in macs) {
                    if (matcher(obj, {'nics.*.mac': macs[mac]})) {
                        VM.log.error('Found conflict: ' + obj.uuid
                            + ' already has MAC ' + macs[mac]);
                        conflict = true;
                    }
                }
                for (vrid in vrids) {
                    if (matcher(obj, {'nics.*.vrrp_vrid': vrids[vrid]})) {
                        VM.log.error('Found conflict: ' + obj.uuid
                            + ' already has VRID ' + vrids[vrid]);
                        conflict = true;
                    }
                }
                cb();
            });
        }, function (e) {
            if (e) {
                callback(e);
            } else {
                VM.log.debug('returning from conflict check');
                callback(null, conflict);
            }
        });
    });
}

function lookupInvalidNicTags(nics, callback) {
    var etherstubs = [];
    var nic_tags = {};

    if (!nics || nics.length === 0) {
        callback();
        return;
    }

    async.parallel([
        function (cb) {
            dladm.showEtherstub(null, VM.log, function (err, stubs) {
                if (err) {
                    cb(err);
                } else {
                    etherstubs = stubs;
                    cb();
                }
            });
        }, function (cb) {
            VM.getSysinfo(function (err, sysinfo) {
                if (err) {
                    cb(err);
                } else {
                    var nic;
                    var tag;
                    for (nic in sysinfo['Network Interfaces']) {
                        nic = sysinfo['Network Interfaces'][nic];
                        for (tag in nic['NIC Names']) {
                            nic_tags[nic['NIC Names'][tag]] = 1;
                        }
                    }
                    cb();
                }
            });
        }
    ], function (err, results) {
        if (err) {
            callback(err);
            return;
        }

        var nic;
        for (nic in nics) {
            nic = nics[nic];
            if (!nic.hasOwnProperty('nic_tag')) {
                continue;
            }
            if (!nic_tags.hasOwnProperty(nic.nic_tag)
                && (etherstubs.indexOf(nic.nic_tag) === -1)) {
                callback(new Error('Invalid nic tag "' + nic.nic_tag + '"'));
                return;
            }
        }

        callback();
        return;
    });
}

// create a new zvol for a VM
function createVolume(volume, callback)
{
    var size;

    VM.log.debug('creating volume ' + JSON.stringify(volume));

    if (volume.hasOwnProperty('image_size')) {
        size = volume.image_size;
    } else if (volume.hasOwnProperty('size')) {
        size = volume.size;
    } else {
        callback(new Error('FATAL: createVolume(' + JSON.stringify(volume)
            + '): ' + 'has no size or image_size'));
        return;
    }

    async.series([
        function (cb) {
            var options;
            var snapshot;
            var target;

            target = volume.zpool + '/' + volume.uuid;
            if (volume.hasOwnProperty('image_uuid')) {
                // This volume is from a template/dataset/image so we create
                // it as a clone of a snapshot of the original.
                snapshot = volume.zpool + '/' + volume.image_uuid
                    + '@' + volume.uuid;

                zfs(['snapshot', snapshot], function (err, fds) {
                    if (err) {
                        cb(err);
                    } else {
                        options = ['clone', '-F'];
                        if (volume.hasOwnProperty('compression')) {
                            options.push('-o', 'compression='
                                + volume.compression);
                        }
                        if (volume.hasOwnProperty('block_size')) {
                            options.push('-o', 'volblocksize='
                                + volume.block_size);
                        }
                        options.push(snapshot, target);
                        zfs(options, function (e) {
                            if (e) {
                                cb(e);
                            } else {
                                volume.path = '/dev/zvol/rdsk/' + target;
                                cb();
                            }
                        });
                    }
                });
            } else {
                // This volume is not from a template/dataset/image so we create
                // a blank new zvol for it.
                options = ['create'];
                if (volume.hasOwnProperty('compression')) {
                    options.push('-o', 'compression='
                        + volume.compression);
                }
                if (volume.hasOwnProperty('block_size')) {
                    options.push('-o', 'volblocksize='
                        + volume.block_size);
                }
                options.push('-o', 'refreservation=none', '-V', size + 'M',
                    target);
                zfs(options, function (err, fds) {
                    if (err) {
                        cb(err);
                    } else {
                        volume.path = '/dev/zvol/rdsk/' + target;
                        cb();
                    }
                });
            }
        }
    ], function (err, results) {
        callback(err);
    });
}

// Create all the volumes for a given VM property set
function createVolumes(payload, callback)
{
    var createme = [];
    var d;
    var disk;
    var disk_idx = 0;
    var used_disk_indexes = [];

    VM.log.debug('creating volumes: ' + JSON.stringify(payload.add_disks));

    if (payload.hasOwnProperty('used_disk_indexes')) {
        used_disk_indexes = payload.used_disk_indexes;
    }

    for (disk in payload.add_disks) {
        if (payload.add_disks.hasOwnProperty(disk)) {
            d = payload.add_disks[disk];

            // we don't create CDROM devices or disk devices which have the
            // nocreate: true property.
            if (d.media !== 'cdrom' && !d.nocreate) {
                // skip to the next unused one.
                while (used_disk_indexes.indexOf(disk_idx) !== -1) {
                    disk_idx++;
                }

                d.index = disk_idx;
                d.uuid = payload.uuid + '-disk' + disk_idx;
                used_disk_indexes.push(Number(disk_idx));
                if (!d.hasOwnProperty('zpool')) {
                    d.zpool = payload.zpool;
                }
                createme.push(d);
            }
        }
    }

    // create all the volumes we found that we need.
    async.forEachSeries(createme, createVolume, function (err) {
        if (err) {
            callback(err);
        } else {
            callback();
        }
    });
}

// writes a Zone's metadata JSON to /zones/<uuid>/config/metadata.json
// and /zones/<uuid>/config/tags.json.
function updateMetadata(vmobj, payload, callback)
{
    var mdata_filename;
    var tags_filename;
    var key;
    var mdata = {};
    var imdata = {};
    var cmdata = {};
    var tags = {};
    var zonepath;

    if (vmobj.hasOwnProperty('zonepath')) {
        zonepath = vmobj.zonepath;
    } else if (vmobj.hasOwnProperty('zpool')
        && vmobj.hasOwnProperty('zonename')) {

        zonepath = '/' + vmobj.zpool + '/' + vmobj.zonename;
    } else {
        callback(new Error('unable to find zonepath for '
            + JSON.stringify(vmobj)));
        return;
    }

    // paths are under zonepath but not zoneroot
    mdata_filename = zonepath + '/config/metadata.json';
    tags_filename = zonepath + '/config/tags.json';

    // customer_metadata
    for (key in vmobj.customer_metadata) {
        if (vmobj.customer_metadata.hasOwnProperty(key)) {
            cmdata[key] = vmobj.customer_metadata[key];
            if (payload.hasOwnProperty('remove_customer_metadata')
                && payload.remove_customer_metadata.indexOf(key) !== -1) {

                // in the remove_* list, don't load it.
                delete cmdata[key];
            }
        }
    }

    for (key in payload.set_customer_metadata) {
        if (payload.set_customer_metadata.hasOwnProperty(key)) {
            cmdata[key] = payload.set_customer_metadata[key];
        }
    }

    // internal_metadata
    for (key in vmobj.internal_metadata) {
        if (vmobj.internal_metadata.hasOwnProperty(key)) {
            imdata[key] = vmobj.internal_metadata[key];
            if (payload.hasOwnProperty('remove_internal_metadata')
                && payload.remove_internal_metadata.indexOf(key) !== -1) {

                // in the remove_* list, don't load it.
                delete imdata[key];
            }
        }
    }

    for (key in payload.set_internal_metadata) {
        if (payload.set_internal_metadata.hasOwnProperty(key)) {
            imdata[key] = payload.set_internal_metadata[key];
        }
    }

    // same thing for tags
    for (key in vmobj.tags) {
        if (vmobj.tags.hasOwnProperty(key)) {
            tags[key] = vmobj.tags[key];
            if (payload.hasOwnProperty('remove_tags')
                && payload.remove_tags.indexOf(key) !== -1) {

                // in the remove_* list, don't load it.
                delete tags[key];
            }
        }
    }

    for (key in payload.set_tags) {
        if (payload.set_tags.hasOwnProperty(key)) {
            tags[key] = payload.set_tags[key];
        }
    }

    mdata = {'customer_metadata': cmdata, 'internal_metadata': imdata};
    fs.writeFile(mdata_filename, JSON.stringify(mdata, null, 2),
        function (err) {
            if (err) {
                callback(err);
            } else {
                VM.log.debug('wrote metadata to ' + mdata_filename);
                fs.writeFile(tags_filename, JSON.stringify(tags, null, 2),
                    function (e) {
                        if (e) {
                            callback(e);
                        } else {
                            VM.log.debug('wrote tags to' + tags_filename);
                            callback();
                        }
                    }
                );
            }
        }
    );
}

function saveMetadata(payload, callback)
{
    var protovm = {};

    if (!payload.hasOwnProperty('zonepath')
        || !payload.hasOwnProperty('zpool')
        || !payload.hasOwnProperty('zonename')) {

        callback(new Error('saveMetadata payload is missing zone '
            + 'properties.'));
        return;
    }

    protovm.zonepath = payload.zonepath;
    protovm.zpool = payload.zpool;
    protovm.zonename = payload.zonename;
    protovm.customer_metadata = {};
    protovm.tags = {};

    if (payload.hasOwnProperty('tags')) {
        payload.set_tags = payload.tags;
        delete payload.tags;
    }
    if (payload.hasOwnProperty('customer_metadata')) {
        payload.set_customer_metadata = payload.customer_metadata;
        delete payload.customer_metadata;
    }
    if (payload.hasOwnProperty('internal_metadata')) {
        payload.set_internal_metadata = payload.internal_metadata;
        delete payload.internal_metadata;
    }

    updateMetadata(protovm, payload, callback);
}

function createVM(payload, callback)
{
    async.series([
        function (cb) {
            if (!payload.create_only) {
                // progress(2, 'checking required datasets');
                checkDatasets(payload, cb);
            } else {
                cb();
            }
        }, function (cb) {
            if (!payload.create_only) {
                // progress(29, 'creating volumes');
                createVolumes(payload, cb);
            } else {
                cb();
            }
        }, function (cb) {
            // progress(51, 'creating zone container');
            createZone(payload, cb);
        }
    ], function (err, results) {
        if (err) {
            callback(err);
        } else {
            callback(null, results);
        }
    });
}

function cleanupMessyDataset(zonepath, brand, callback)
{
    var command;
    var zoneroot = path.join(zonepath, '/root');

    // Some datasets leave junk behind that they shouldn't or is only there for
    // old platforms. This cleans up junk files to ensure the zone comes up a
    // bit cleaner.
    fs.exists(zoneroot, function (exists) {
        if (exists) {
            command = 'rm -rf '
                + zoneroot + '/var/adm/utmpx '
                + zoneroot + '/var/adm/wtmpx '
                + zoneroot + '/var/svc/log/*.log '
                + zoneroot + '/var/svc/mdata '
                + zoneroot + '/var/svc/manifest/mdata.xml ';

            if (! BRAND_OPTIONS[brand].features.zoneinit) {
                // eg. joyent-minimal (don't need zoneinit)
                command = command + zoneroot + '/root/zone* ';
            }

            command = command + '&& touch ' + zoneroot + '/var/adm/wtmpx';
            VM.log.debug(command);
            exec(command, function (error, stdout, stderr) {
                VM.log.debug('stdout: ' + stdout);
                VM.log.debug('stderr: ' + stderr);
                callback(error);
            });
        } else {
            callback(new Error(zoneroot
                + ' does not exist! Cannot cleanup dataset.'));
        }
    });
}

function writeZoneNetfiles(payload, callback)
{
    var hostname;
    var n;
    var nic;
    var primary_found = false;

    VM.log.info('Writing network files to zone root');

    for (nic in payload.add_nics) {
        if (payload.add_nics.hasOwnProperty(nic)) {
            n = payload.add_nics[nic];

            if (n.ip != 'dhcp') {
                fs.writeFileSync(payload.zonepath + '/root/etc/hostname.'
                    + n.interface, n.ip + ' netmask ' + n.netmask
                    + ' up' + '\n');
            }

            if (n.hasOwnProperty('primary') && !primary_found) {
                // only allow one primary network
                primary_found = true;
                if (n.hasOwnProperty('gateway')) {
                    fs.writeFileSync(payload.zonepath
                        + '/root/etc/defaultrouter', n.gateway + '\n');
                }
                if (n.ip == 'dhcp') {
                    fs.writeFileSync(payload.zonepath + '/root/etc/dhcp.'
                        + n.interface, '');
                }
            }
        }
    }

    // It's possible we don't have zonename or hostname set because of the
    // ordering of adding the UUID. In any case, we'll have at least a uuid
    // here.
    if (payload.hasOwnProperty('hostname')) {
        hostname = payload.hostname;
    } else if (payload.hasOwnProperty('zonename')) {
        hostname = payload.zonename;
    } else {
        hostname = payload.uuid;
    }

    fs.writeFileSync(payload.zonepath + '/root/etc/nodename', hostname + '\n');

    callback();
}

/*
 * XXX do we still need this?
 *
 * This writes out the zoneconfig file that is used by the zoneinit service in
 * joyent branded zones' datasets.
 *
 */
function writeZoneconfig(payload, callback)
{
    var data;
    var hostname;
    var n;
    var nic;

    VM.log.info('Writing config for zoneinit');

    if (payload.hasOwnProperty('hostname')) {
        hostname = payload.hostname;
    } else {
        hostname = payload.zonename;
    }

    data = 'TEMPLATE_VERSION=0.0.1\n'
        + 'ZONENAME=' + payload.zonename + '\n'
        + 'HOSTNAME=' + hostname + '.' + payload.dns_domain + '\n'
        + 'TMPFS=' + payload.tmpfs + 'm\n';

    if (payload.hasOwnProperty('add_nics') && payload.add_nics[0]) {

        if (payload.add_nics[0] && payload.add_nics[0].ip != 'dhcp') {
            data = data + 'PUBLIC_IP=' + payload.add_nics[0].ip + '\n';
        }
        if (payload.add_nics[1] && payload.add_nics[1].ip != 'dhcp') {
            data = data + 'PRIVATE_IP=' + payload.add_nics[1].ip + '\n';
        } else if (payload.add_nics[0] && payload.add_nics[0].ip != 'dhcp') {
            // zoneinit uses private_ip for /etc/hosts, we want to
            // make that same as public, if there's no actual private.
            data = data + 'PRIVATE_IP=' + payload.add_nics[0].ip + '\n';
        }
    }

    if (payload.hasOwnProperty('resolvers')) {
        // zoneinit appends to resolv.conf rather than overwriting, so just
        // add to the zoneconfig and let zoneinit handle it
        data = data + 'RESOLVERS="' + payload.resolvers.join(' ') + '"\n';
    }

    for (nic in payload.add_nics) {
        if (payload.add_nics.hasOwnProperty(nic)) {
            n = payload.add_nics[nic];
            data = data + n.interface.toUpperCase() + '_MAC=' + n.mac + '\n'
                + n.interface.toUpperCase() + '_INTERFACE='
                + n.interface.toUpperCase() + '\n';

            if (n.ip != 'dhcp') {
                data = data + n.interface.toUpperCase() + '_IP=' + n.ip + '\n'
                    + n.interface.toUpperCase() + '_NETMASK='
                    + n.netmask + '\n';
            }
        }
    }

    fs.writeFileSync(payload.zonepath
        + '/root/var/svc/log/system-zoneinit:default.log', '');

    VM.log.debug('writing ' + JSON.stringify(data) + ' to /'
        + payload.zpool + '/' + payload.zonename);

    fs.writeFile('/' + payload.zpool + '/' + payload.zonename
        + '/root/root/zoneconfig', data, function (err, result) {

        if (err) {
            callback(err);
        } else {
            callback();
        }
    });
}

function zonecfg(args, callback)
{
    var cmd = '/usr/sbin/zonecfg';

    VM.log.debug(cmd + ' ' + args.join(' '));
    execFile(cmd, args, function (error, stdout, stderr) {
        if (error) {
            // TODO log here.
            callback(error, {'stdout': stdout, 'stderr': stderr});
        } else {
            callback(null, {'stdout': stdout, 'stderr': stderr});
        }
    });
}

function zonecfgFile(data, args, callback)
{
    var tmpfile = '/tmp/zonecfg.' + process.pid + '.tmp';

    fs.writeFile(tmpfile, data, function (err, result) {
        if (err) {
            // On failure we don't delete the tmpfile so we can debug it.
            callback(err);
        } else {
            args.push('-f');
            args.push(tmpfile);

            zonecfg(args, function (e, fds) {
                if (e) {
                    // keep temp file around for investigation
                    callback(e, fds);
                } else {
                    fs.unlink(tmpfile, function () {
                        callback(null, fds);
                    });
                }
            });
        }
    });
}

function zoneadm(args, callback)
{
    var cmd = '/usr/sbin/zoneadm';

    VM.log.debug(cmd + ' ' + args.join(' '));
    execFile(cmd, args, function (error, stdout, stderr) {
        if (error) {
            callback(error, {'stdout': stdout, 'stderr': stderr});
        } else {
            callback(null, {'stdout': stdout, 'stderr': stderr});
        }
    });
}

function zfs(args, callback)
{
    var cmd = '/usr/sbin/zfs';

    VM.log.debug(cmd + ' ' + args.join(' '));
    execFile(cmd, args, function (error, stdout, stderr) {
        if (error) {
            callback(error, {'stdout': stdout, 'stderr': stderr});
        } else {
            callback(null, {'stdout': stdout, 'stderr': stderr});
        }
    });
}

exports.getSysinfo = function (args, callback)
{
    var cmd = '/usr/bin/sysinfo';

    ensureLogging('getSysinfo', false);

    if (typeof (args) === 'function') {
        callback = args;
        args = [];
    }

    VM.log.debug(cmd + ' ' + args.join(' '));
    execFile(cmd, args, function (error, stdout, stderr) {
        var sysinfo;

        if (error) {
            callback(error, {'stdout': stdout, 'stderr': stderr});
        } else {
            try {
                sysinfo = JSON.parse(stdout.toString());
            } catch (e) {
                sysinfo = {};
            }
            callback(null, sysinfo);
        }
    });
};

/*
 * This watches zone transitions and calls callback when specified
 * state is reached.  Optionally you can set a timeout which will
 * call your callback when the timeout occurs whether the transition
 * has happened or not.
 *
 * payload needs to have at least .zonename
 *
 */
exports.waitForZoneState = function (payload, state, options, callback) {
    var sysevent_state;
    var timeout;
    var timeout_secs = PROVISION_TIMEOUT;
    var watcher;

    ensureLogging('waitForZoneState', false);

    if (options && options.hasOwnProperty('timeout')) {
        timeout_secs = options.timeout;
    }

    sysevent_state = state;
    if (state === 'installed') {
        // Apparently the zone status 'installed' equals sysevent status
        // 'uninitialized'
        sysevent_state = 'uninitialized';
    }

    function done() {
        if (timeout) {
            clearTimeout(timeout);
            timeout = null;
        }
    }

    function handler(obj) {
        VM.log.debug('handler got: ' + JSON.stringify(obj));
        if (obj.zonename !== payload.zonename) {
            done();
            callback(new Error('hander got state obj for wrong zone! ('
                + obj.zonename + ' != ' + payload.zonename + ')'));
            return;
        }

        if (obj.newstate === sysevent_state) {
            // Load again to confirm
            VM.lookup({'zonename': obj.zonename}, {'full': true},
                function (err, res) {
                    var handler_retry;

                    if (err) {
                        watcher.cleanup();
                        done();
                        callback(err);
                        return;
                    }

                    if (res.length !== 1) {
                        watcher.cleanup();
                        done();
                        callback(new Error('lookup could no find VM '
                            + obj.zonename));
                        return;
                    }

                    if (res[0].hasOwnProperty('zone_state')
                        && res[0].zone_state === state) {

                        // found the state we're looking for, success!
                        VM.log.debug('saw zone go to ' + obj.newstate + ' ('
                            + state + ') calling callback()');
                        watcher.cleanup();
                        done();
                        callback();
                    } else if (timeout) {
                        if (!handler_retry) {
                            handler_retry = setTimeout(function () {
                                if (timeout) {
                                    // try again if wait timeout is still set
                                    handler(obj);
                                }
                                handler_retry = null;
                            }, 1000);
                            VM.log.debug('zone state after lookup: '
                                + res[0].zone_state + ', still waiting');
                        } else {
                            VM.log.debug('zone in wrong state but we already'
                                + ' have a handler running');
                        }
                    } else {
                        // no timeout set and we're not at the correct state
                        VM.log.error('failed to reach state: ' + state);
                        callback(new Error('failed to reach state: ' + state));
                    }
                }
            );
        }
    }

    watcher = watchZoneTransitions(payload, handler, callback);

    timeout = setTimeout(function () {
        var err;

        done();
        watcher.cleanup();
        err = new Error('timed out waiting for zone to transition to ' + state);
        err.code = 'ETIMEOUT';
        callback(err);
    }, timeout_secs * 1000);
};

//
// handler() will be called with an object describing the transition for any
// transitions seen (after any filtering).
//
//  vmobj here is an object with {"uuid": ..., "zonename": ...}, if this is
//  non-null, it limits calls to handler() to transitions for this VM.
//
function watchZoneTransitions(vmobj, handler, callback) {
    var buffer = '';
    var chunks;
    var cleanup;
    var watcher;
    var watcher_pid;

    // verify we don't have bad options
    if (vmobj) {
        if (!vmobj.hasOwnProperty('zonename')
            || !vmobj.hasOwnProperty('uuid')) {

            callback(new Error('vmobj must have both uuid and zonename'));
            return ({});
        }
    }

    VM.log.debug('/usr/vm/sbin/zoneevent');
    watcher = spawn('/usr/vm/sbin/zoneevent', [], {'customFds': [-1, -1, -1]});
    VM.log.info('zoneevent running with pid ' + watcher.pid);
    watcher_pid = watcher.pid;

    cleanup = function () {
        if (watcher) {
            VM.log.debug('zoneevent watcher ' + watcher_pid
                + ' cleanup called');
            watcher.removeAllListeners('exit'); // so we don't fail on our kill
            watcher.kill();
            watcher = null;
        }
    };

    watcher.stdout.on('data', function (data) {
        var chunk;
        var obj;
        var prev_msg;

        buffer += data.toString();
        chunks = buffer.split('\n');
        while (chunks.length > 1) {
            chunk = chunks.shift();
            obj = JSON.parse(chunk);

            if (obj === prev_msg) {
                // Note: sometimes sysevent emits multiple events for the same
                // status, we only want the first one here because calling
                // handler() twice in a row for the same event is not nice.
                VM.log.debug('duplicate zoneevent message! '
                    + JSON.stringify(obj));
            } else if (!vmobj || obj.zonename === vmobj.zonename) {
                VM.log.debug('zoneevent said: ' + JSON.stringify(obj));
                handler(obj);
            }
        }
        buffer = chunks.pop();
    });
    watcher.stdin.end();

    watcher.on('exit', function (code) {
        VM.log.error('zoneevent watcher ' + watcher_pid + ' exited: ',
            JSON.stringify(code));
        cleanup();
        callback(new Error('zoneevent watcher exited prematurely with code: '
            + code));
    });

    return ({'cleanup': cleanup});
}

function fixPayloadMemory(payload, vmobj)
{
    var brand;
    var max_locked;
    var max_phys;
    var min_overhead;
    var ram;

    if (vmobj.hasOwnProperty('brand')) {
        brand = vmobj.brand;
    } else if (payload.hasOwnProperty('brand')) {
        brand = payload.brand;
    }

    if (BRAND_OPTIONS[brand].features.default_memory_overhead
        && payload.hasOwnProperty('ram')
        && !payload.hasOwnProperty('max_physical_memory')) {

        // For now we add overhead to the memory caps for KVM zones, this
        // is for the qemu process itself.  Since customers don't have direct
        // access to zone memory, this exists mostly to protect against bugs.
        payload.max_physical_memory = (payload.ram
            + BRAND_OPTIONS[brand].features.default_memory_overhead);
    } else if (payload.hasOwnProperty('ram')
        && !payload.hasOwnProperty('max_physical_memory')) {

        payload.max_physical_memory = payload.ram;
    }

    if (payload.hasOwnProperty('max_physical_memory')) {
        if (!payload.hasOwnProperty('max_locked_memory')) {
            if (vmobj.hasOwnProperty('max_locked_memory')
                && vmobj.hasOwnProperty('max_physical_memory')) {

                // we don't have a new value, so first try to keep the same
                // delta that existed before btw. max_phys and max_locked
                payload.max_locked_memory = payload.max_physical_memory
                    - (vmobj.max_physical_memory - vmobj.max_locked_memory);
            } else {
                // existing obj doesn't have max_locked, add one now
                payload.max_locked_memory = payload.max_physical_memory;
            }
        }

        if (!payload.hasOwnProperty('max_swap')) {
            if (vmobj.hasOwnProperty('max_swap')
                && vmobj.hasOwnProperty('max_physical_memory')) {

                // we don't have a new value, so first try to keep the same
                // delta that existed before btw. max_phys and max_swap
                payload.max_swap = payload.max_physical_memory
                    + (vmobj.max_swap - vmobj.max_physical_memory);
            } else {
                // existing obj doesn't have max_swap, add one now
                payload.max_swap = payload.max_physical_memory;
            }
        }
    }

    // if we're updating tmpfs it must be lower than our new max_physical or
    // if we're not also changing max_physical, it must be lower than the
    // current one.
    if (payload.hasOwnProperty('tmpfs')) {
        if (payload.hasOwnProperty('max_physical_memory')
            && (Number(payload.tmpfs)
                > Number(payload.max_physical_memory))) {

            payload.tmpfs = payload.max_physical_memory;
        } else if (Number(payload.tmpfs)
            > Number(vmobj.max_physical_memory)) {

            payload.tmpfs = vmobj.max_physical_memory;
        }
    }

    if (payload.hasOwnProperty('max_physical_memory')
        && BRAND_OPTIONS[brand].features.use_tmpfs
        && !payload.hasOwnProperty('tmpfs')) {

        if (vmobj.hasOwnProperty('max_physical_memory')
            && vmobj.hasOwnProperty('tmpfs')) {

            // change tmpfs to be the same ratio of ram as before
            payload.tmpfs = ((vmobj.tmpfs / vmobj.max_physical_memory)
                * payload.max_physical_memory);
            payload.tmpfs = Number(payload.tmpfs).toFixed();
        } else {
            // tmpfs must be < max_physical_memory, if not: pretend it was
            payload.tmpfs = payload.max_physical_memory;
        }
    }

    // now that we've possibly adjusted target values, lower/raise values to
    // satisify max/min.

    min_overhead = BRAND_OPTIONS[brand].features.min_memory_overhead;
    if (min_overhead) {
        ram = payload.hasOwnProperty('ram') ? payload.ram : vmobj.ram;
        max_phys = payload.hasOwnProperty('max_physical_memory')
            ? payload.max_physical_memory : vmobj.max_physical_memory;
        max_locked = payload.hasOwnProperty('max_locked_memory')
            ? payload.max_locked_memory : vmobj.max_locked_memory;

        if ((ram + min_overhead) > max_phys) {
            payload.max_physical_memory = (ram + min_overhead);
        }
        if ((ram + min_overhead) > max_locked) {
            payload.max_locked_memory = (ram + min_overhead);
        }
    }

    if (payload.hasOwnProperty('max_locked_memory')) {
        if (payload.hasOwnProperty('max_physical_memory')) {
            if (payload.max_locked_memory > payload.max_physical_memory) {
                VM.log.warn('max_locked_memory (' + payload.max_locked_memory
                    + ') > max_physical_memory (' + payload.max_physical_memory
                    + ') clamping to ' + payload.max_physical_memory);
                payload.max_locked_memory = payload.max_physical_memory;
            }
        } else if (vmobj.hasOwnProperty('max_physical_memory')) {
            // new payload doesn't have a max_physical, so clamp to vmobj's
            if (payload.max_locked_memory > vmobj.max_physical_memory) {
                VM.log.warn('max_locked_memory (' + payload.max_locked_memory
                    + ') > vm.max_physical_memory (' + vmobj.max_physical_memory
                    + ') clamping to ' + vmobj.max_physical_memory);
                payload.max_locked_memory = vmobj.max_physical_memory;
            }
        }
    }

    if (payload.hasOwnProperty('max_swap')) {
        if (payload.hasOwnProperty('max_physical_memory')) {
            if (payload.max_swap < payload.max_physical_memory) {
                VM.log.warn('max_swap (' + payload.max_swap
                    + ') < max_physical_memory (' + payload.max_physical_memory
                    + ') raising to ' + payload.max_physical_memory);
                payload.max_swap = payload.max_physical_memory;
            }
        } else if (vmobj.hasOwnProperty('max_physical_memory')) {
            // new payload doesn't have a max_physical, so raise to vmobj's
            if (payload.max_swap < vmobj.max_physical_memory) {
                VM.log.warn('max_swap (' + payload.max_swap
                    + ') < vm.max_physical_memory (' + vmobj.max_physical_memory
                    + ') raising to ' + vmobj.max_physical_memory);
                payload.max_swap = vmobj.max_physical_memory;
            }
        }
    }
}

// generate a new UUID if payload doesn't have one (also ensures that this uuid
// does not already belong to a zone).
function createZoneUUID(payload, callback)
{
    var uuid;

    if (payload.hasOwnProperty('uuid')) {
        // Ensure that the uuid is not already used.
        getZoneRecords(null, function (err, records) {
            if (err) {
                callback(err);
            } else {
                if (records.hasOwnProperty(payload.uuid)) {
                    callback(new Error('vm with UUID ' + payload.uuid
                        + ' already exists.'));
                } else {
                    callback(null, payload.uuid);
                }
            }
        });
    } else {
        VM.log.debug('/usr/bin/uuid -v 4');
        execFile('/usr/bin/uuid', ['-v', '4'], function (err, stdout, stderr) {
            if (err) {
                callback(err);
                return;
            }

            // chomp trailing spaces and newlines
            uuid = stdout.toString().replace(/\s+$/g, '');
            payload.uuid = uuid;
            // This VM didn't have a uuid before (we just added one)
            // so normalize couldn't set the zonename, do so now.
            if (!payload.hasOwnProperty('zonename')
                || payload.zonename === undefined) {

                payload.zonename = payload.uuid;
                VM.log.info('createZoneUUID() set payload.zonename = '
                    + payload.zonename);
            }
            VM.log.info('createZoneUUID() generated uuid ' + uuid);
            getZoneRecords(null, function (e, records) {
                if (e) {
                    callback(e);
                } else {
                    if (records.hasOwnProperty(payload.uuid)) {
                        callback(new Error('vm with UUID ' + payload.uuid
                            + 'already exists.'));
                    } else {
                        callback(null, payload.uuid);
                    }
                }
            });
        });
    }
}

function applyZoneDefaults(payload)
{
    var allowed;
    var disk;
    var disks;
    var n;
    var nic;
    var nics;
    var zvol;

    VM.log.debug('applying zone defaults');

    if (!payload.hasOwnProperty('owner_uuid')) {
        // We assume that this all-zero uuid can be treated as 'admin'
        payload.owner_uuid = '00000000-0000-0000-0000-000000000000';
    }

    if (!payload.hasOwnProperty('autoboot')) {
        payload.autoboot = true;
    }

    if (!payload.hasOwnProperty('brand')) {
        payload.brand = 'joyent';
    }

    if (!payload.hasOwnProperty('zpool')) {
        payload.zpool = 'zones';
    }

    if (!payload.hasOwnProperty('dns_domain')) {
        payload.dns_domain = 'local';
    }

    if (!payload.hasOwnProperty('cpu_shares')) {
        payload.cpu_shares = 100;
    } else {
        if (payload.cpu_shares > 65535) {
            VM.log.info('capping cpu_shares at 64k (was: '
                + payload.cpu_shares + ')');
            payload.cpu_shares = 65535; // max is 64K
        }
    }

    if (!payload.hasOwnProperty('zfs_io_priority')) {
        payload.zfs_io_priority = 100;
    }

    if (!payload.hasOwnProperty('max_lwps')) {
        payload.max_lwps = 2000;
    }

    // We need to set the RAM here because we use it as the default for
    // the max_physical_memory below. If we've set max_phys and we're not
    // KVM, we'll use that instead of ram anyway.
    if (!payload.hasOwnProperty('ram')) {
        payload.ram = 256;
    }

    fixPayloadMemory(payload, {});

    allowed = BRAND_OPTIONS[payload.brand].allowed_properties;
    if (allowed.hasOwnProperty('vcpus') && !payload.hasOwnProperty('vcpus')) {
        payload.vcpus = 1;
    }

    if (BRAND_OPTIONS[payload.brand].features.use_tmpfs
        && (!payload.hasOwnProperty('tmpfs')
            || (Number(payload.tmpfs) > Number(payload.max_physical_memory)))) {

        payload.tmpfs = payload.max_physical_memory;
    }

    if (!payload.hasOwnProperty('limit_priv')) {
        // note: the limit privs are going to be added to the brand and
        // shouldn't need to be set here by default when that's done.
        if (BRAND_OPTIONS[payload.brand].features.limit_priv) {
            payload.limit_priv
                = BRAND_OPTIONS[payload.brand].features.limit_priv.join(',');
        } else {
            payload.limit_priv = 'default';
        }
    }

    if (!payload.hasOwnProperty('quota')) {
        payload.quota = '10'; // in GiB
    }

    if (!payload.hasOwnProperty('billing_id')) {
        payload.billing_id = '00000000-0000-0000-0000-000000000000';
    }

    if (payload.hasOwnProperty('add_disks')) {
        // update
        disks = payload.add_disks;
    } else if (payload.hasOwnProperty('disks')) {
        disks = payload.disks;
    } else {
        // no disks at all
        disks = [];
    }

    for (disk in disks) {
        if (disks.hasOwnProperty(disk)) {
            zvol = disks[disk];
            if (!zvol.hasOwnProperty('model')
                && payload.hasOwnProperty('disk_driver')) {

                zvol.model = payload.disk_driver;
            }
            if (!zvol.hasOwnProperty('media')) {
                zvol.media = 'disk';
            }
        }
    }

    if (payload.hasOwnProperty('add_nics')) {
        // update
        nics = payload.add_nics;
    } else if (payload.hasOwnProperty('nics')) {
        nics = payload.nics;
    } else {
        // no disks at all
        nics = [];
    }

    for (nic in nics) {
        if (nics.hasOwnProperty(nic)) {
            n = nics[nic];
            if (!n.hasOwnProperty('model')
                && payload.hasOwnProperty('nic_driver')) {

                n.model = payload.nic_driver;
            }
        }
    }
}

function validRecordSize(candidate)
{
    if (candidate < 512) {
        // too low
        return (false);
    } else if (candidate > 131072) {
        // too high
        return (false);
    } else if ((candidate & (candidate - 1)) !== 0) {
        // not a power of 2
        return (false);
    }

    return (true);
}

// This function gets called for both create and update to check that payload
// properties are reasonable. If vmobj is null, create is assumed, otherwise
// update is assumed.
function checkPayloadProperties(payload, vmobj, callback)
{
    var array_fields = [
        'add_nics', 'update_nics', 'remove_nics',
        'add_disks', 'update_disks', 'remove_disks',
        'add_filesystems', 'update_filesystems', 'remove_filesystems'
    ];
    var changed_nics = [];
    var current_ips = [];
    var current_macs = [];
    var current_primary_ips = [];
    var current_vrids = [];
    var disk;
    var field;
    var filesys;
    var i;
    var ips = [];
    var mac;
    var macs = [];
    var m;
    var n;
    var nic;
    var nics_result = {};
    var nic_fields = ['add_nics', 'update_nics'];
    var only_vrrp_nics = true;
    var primary_nics;
    var prop;
    var props;
    var ram;
    var brand;
    var vrids = [];
    var zvol;

    if (vmobj) {
        brand = vmobj.brand;
    } else if (payload.hasOwnProperty('brand')) {
        brand = payload.brand;
    } else {
        callback(new Error('unable to determine brand for VM'));
    }

    /* check types of fields that should be arrays */
    for (field in array_fields) {
        field = array_fields[field];
        if (payload.hasOwnProperty(field) && ! Array.isArray(payload[field])) {
            callback(new Error(field + ' must be an array.'));
            return;
        }
    }

    if (!vmobj) {
        // This is a CREATE

        if (BRAND_OPTIONS[brand].features.check_avail_ram
            && payload.hasOwnProperty('available_MiB')) {

            if (payload.available_MiB < payload.ram) {
                callback(new Error('VM "ram" value: ' + payload.ram + ' is '
                    + 'greater than' + ' available memory: '
                    + payload.available_MiB));
                return;
            } else {
                VM.log.debug('checkPayloadProperties() memory ok: '
                    + payload.ram + ' requested and have: '
                    + payload.available_MiB);
            }
        }

        // These should have already been enforced
        if (payload.max_locked_memory > payload.max_physical_memory) {
            callback(new Error('max_locked_memory must be <= '
                + 'max_physical_memory'));
            return;
        }
        if (payload.max_swap < payload.max_physical_memory) {
            callback(new Error('max_swap must be >= max_physical_memory'));
            return;
        }

        // We used to use zone_path instead of zonepath, so accept that too.
        if (payload.hasOwnProperty('zone_path')
            && !payload.hasOwnProperty('zonepath')) {

            payload.zonepath = payload.zone_path;
            delete payload.zone_path;
        }
    } else {
        // This is an UPDATE

        // can't update disks of a running VM
        if ((payload.hasOwnProperty('add_disks')
            || payload.hasOwnProperty('remove_disks')
            || payload.hasOwnProperty('update_disks'))) {

            if ((vmobj.state !== 'stopped')
                || (vmobj.state === 'provisioning'
                && vmobj.zone_state !== 'installed')) {

                callback(new Error('updates to disks are only allowed when '
                    + 'state is "stopped", currently: ' + vmobj.state + ' ('
                    + vmobj.zonestate + ')'));
                return;
            }
        }

        // if there's a min_overhead we ensure values are higher than ram.
        if (BRAND_OPTIONS[brand].features.min_memory_overhead) {
            if (payload.hasOwnProperty('ram')) {
                ram = payload.ram;
            } else {
                ram = vmobj.ram;
            }

            // ensure none of these is < ram
            if (payload.hasOwnProperty('max_physical_memory')
                && payload.max_physical_memory < ram) {

                callback(new Error('vm.max_physical_memory ('
                    + payload.max_physical_memory + ') cannot be lower than'
                    + ' vm.ram (' + ram + ')'));
                return;
            }
            if (payload.hasOwnProperty('max_locked_memory')
                && payload.max_locked_memory < ram) {

                callback(new Error('vm.max_locked_memory ('
                    + payload.max_locked_memory + ') cannot be lower than'
                    + ' vm.ram (' + ram + ')'));
                return;
            }
            // This should not be allowed anyway because max_swap will be raised
            // to match max_physical_memory if you set it lower.
            if (payload.hasOwnProperty('max_swap')
                && payload.max_swap < ram) {

                callback(new Error('vm.max_swap ('
                    + payload.max_swap + ') cannot be lower than'
                    + ' vm.ram (' + ram + ')'));
                return;
            }
        }

        /*
         * keep track of current IPs/MACs so we can make sure they're not being
         * duplicated.
         *
         */
        for (nic in vmobj.nics) {
            nic = vmobj.nics[nic];
            if (nic.hasOwnProperty('ip') && nic.ip !== 'dhcp') {
                current_ips.push(nic.ip);
            }
            if (nic.hasOwnProperty('mac')) {
                current_macs.push(nic.mac);
            }
            if (nic.hasOwnProperty('vrrp_vrid')) {
                current_vrids.push(nic.vrrp_vrid);
            }
            if (nic.hasOwnProperty('vrrp_primary_ip')) {
                current_primary_ips.push(nic.vrrp_primary_ip);
            }

            if (nic.hasOwnProperty('mac') || nic.hasOwnProperty('vrrp_vrid')) {
                mac = nic.hasOwnProperty('mac') ? nic.mac
                    : vrrpMAC(nic.vrrp_vrid);
                if (!nics_result.hasOwnProperty(mac)) {
                    nics_result[mac] = nic;
                }
            }
        }
    }

    if (payload.hasOwnProperty('add_disks')) {
        for (disk in payload.add_disks) {
            if (payload.add_disks.hasOwnProperty(disk)) {
                zvol = payload.add_disks[disk];

                // path is only allowed in 2 cases when adding a disk:
                //
                // 1) for cdrom devices
                // 2) when nocreate is specified
                //
                if (zvol.hasOwnProperty('path')) {
                    if (zvol.media !== 'cdrom' && !zvol.nocreate) {
                        callback(new Error('you cannot specify a path for a '
                            + 'disk unless you set nocreate=true'));
                        return;
                    }
                }

                // NOTE: We'll have verified the .zpool argument is a valid
                // zpool using VM.validate() if it's set.

                if (zvol.hasOwnProperty('block_size')
                    && !validRecordSize(zvol.block_size)) {

                    callback(new Error('invalid .block_size(' + zvol.block_size
                        + '), must be 512-131072 and a power of 2.'));
                    return;
                }

                if (zvol.hasOwnProperty('block_size')
                    && zvol.hasOwnProperty('image_uuid')) {

                    callback(new Error('setting both .block_size and '
                        + '.image_uuid on a volume is invalid'));
                }

                if (zvol.hasOwnProperty('compression')) {
                    if (VM.COMPRESSION_TYPES.indexOf(zvol.compression) === -1) {
                        callback(new Error('invalid compression setting for '
                            + 'disk, must be one of: '
                            + VM.COMPRESSION_TYPES.join(', ')));
                    }
                }

                if (!zvol.hasOwnProperty('model')
                    || zvol.model === 'undefined') {

                    callback(new Error('missing .model option for '
                        + 'disk: ' + JSON.stringify(zvol)));
                    return;
                } else if (VM.DISK_MODELS.indexOf(zvol.model) === -1) {
                    callback(new Error('"' + zvol.model + '"'
                        + ' is not a valid disk model. Valid are: '
                        + VM.DISK_MODELS.join(',')));
                    return;
                }
            }
        }
    }

    if (payload.hasOwnProperty('update_disks')) {
        for (disk in payload.update_disks) {
            if (payload.update_disks.hasOwnProperty(disk)) {
                zvol = payload.update_disks[disk];

                if (zvol.hasOwnProperty('compression')) {
                    if (VM.COMPRESSION_TYPES.indexOf(zvol.compression) === -1) {
                        callback(new Error('invalid compression type for '
                            + 'disk, must be one of: '
                            + VM.COMPRESSION_TYPES.join(', ')));
                    }
                }

                if (zvol.hasOwnProperty('block_size')) {
                    callback(new Error('cannot change .block_size for a disk '
                        + 'after creation'));
                    return;
                }
            }
        }
    }

    // If we're receiving, we might not have the filesystem yet
    if (!payload.hasOwnProperty('transition')
        || payload.transition.transition !== 'receiving') {

        for (filesys in payload.filesystems) {
            filesys = payload.filesystems[filesys];
            if (!fs.existsSync(filesys.source)) {
                callback(new Error('missing requested filesystem: '
                    + filesys.source));
                return;
            }
        }
    }

    if (payload.hasOwnProperty('default_gateway')
        && payload.default_gateway !== '') {

        VM.log.warn('DEPRECATED: default_gateway should no longer be used, '
            + 'instead set one NIC primary and use nic.gateway.');
    }

    primary_nics = 0;
    for (field in nic_fields) {
        field = nic_fields[field];
        if (payload.hasOwnProperty(field)) {
            for (nic in payload[field]) {
                if (payload[field].hasOwnProperty(nic)) {
                    n = payload[field][nic];

                    // MAC will always conflict in update, since that's the key
                    if (field === 'add_nics' && n.hasOwnProperty('mac')) {
                        if ((macs.indexOf(n.mac) !== -1)
                            || current_macs.indexOf(n.mac) !== -1) {

                            callback(new Error('Cannot add multiple NICs with '
                                + 'the same MAC: ' + n.mac));
                            return;
                        }
                        macs.push(n.mac);
                    }

                    if (field === 'add_nics' || field === 'update_nics') {
                        if (n.hasOwnProperty('primary')) {
                            if (n.primary !== true) {
                                callback(new Error('invalid value for NIC\'s '
                                    + 'primary flag: ' + n.primary + ' (must be'
                                    + ' true)'));
                                return;
                            }
                            primary_nics++;
                        }
                        changed_nics.push(n);
                    }

                    if (n.hasOwnProperty('ip') && n.ip != 'dhcp') {
                        if (ips.indexOf(n.ip) !== -1
                            || current_ips.indexOf(n.ip) !== -1) {

                            callback(new Error('Cannot add multiple NICs with '
                                + 'the same IP: ' + n.ip));
                            return;
                        }
                        ips.push(n.ip);
                    }

                    if (n.hasOwnProperty('vrrp_vrid')) {
                        if (current_vrids.indexOf(n.vrrp_vrid) !== -1
                            || vrids.indexOf(n.vrrp_vrid) !== -1) {
                            callback(new Error('Cannot add multiple NICs with '
                                + 'the same VRID: ' + n.vrrp_vrid));
                            return;
                        }
                        vrids.push(n.vrrp_vrid);
                    }

                    if (field === 'add_nics'
                        && n.hasOwnProperty('vrrp_vrid')
                        && n.hasOwnProperty('mac')) {
                        callback(
                            new Error('Cannot set both mac and vrrp_vrid'));
                        return;
                    }

                    if (n.hasOwnProperty('vrrp_primary_ip')) {
                        current_primary_ips.push(n.vrrp_primary_ip);
                    }

                    if (BRAND_OPTIONS[brand].features.model_required
                        && n.field === 'add_nics'
                        && (!n.hasOwnProperty('model')
                        || n.model === 'undefined')) {

                        callback(new Error('missing .model option for NIC: '
                            + JSON.stringify(n)));
                        return;
                    }

                    if (n.hasOwnProperty('mac')
                        || n.hasOwnProperty('vrrp_vrid')) {
                        mac = n.hasOwnProperty('mac') ? n.mac
                            : vrrpMAC(n.vrrp_vrid);
                        if (nics_result.hasOwnProperty(mac)) {
                            for (var p in n) {
                                nics_result[mac][p] = n[p];
                            }
                        } else {
                            nics_result[mac] = n;
                        }
                    }
                }
            }
        }
    }

    if (payload.hasOwnProperty('remove_nics')) {
        for (m in payload.remove_nics) {
            m = payload.remove_nics[m];
            n = nics_result[m];
            if (!n) {
                continue;
            }
            if (n.hasOwnProperty('ip') && n.ip != 'dhcp') {
                i = ips.indexOf(n.ip);
                if (i !== -1) {
                    ips.splice(i, 1);
                }
                i = current_ips.indexOf(n.ip);
                if (i !== -1) {
                    current_ips.splice(i, 1);
                }
            }
            delete nics_result[m];
        }
    }

    // nics_result now has the state of the nics after the update - now check
    // properties that depend on each other or on other nics
    for (n in nics_result) {
        n = nics_result[n];
        if (n.hasOwnProperty('vrrp_vrid')) {
            if (n.hasOwnProperty('ip')
                && current_primary_ips.indexOf(n.ip) !== -1) {
                callback(
                    new Error(
                        'Cannot set vrrp_primary_ip to the IP of a VRRP nic'));
                return;
            }

            if (!n.hasOwnProperty('vrrp_primary_ip')) {
                callback(new Error(
                    'vrrp_vrid set but not vrrp_primary_ip'));
                return;
            }
        } else {
            only_vrrp_nics = false;
        }
    }

    if (only_vrrp_nics && Object.keys(nics_result).length !== 0) {
        callback(new Error('VM cannot contain only VRRP nics'));
        return;
    }

    for (i in current_primary_ips) {
        i = current_primary_ips[i];
        if ((current_ips.indexOf(i) === -1)
            && (ips.indexOf(i) === -1)) {
            callback(new Error(
                'vrrp_primary_ip must belong to the same VM'));
            return;
        }
    }

    // Since we always need a primary nic, don't allow a value other than true
    // for primary flag. Also ensure we're not trying to set primary for more
    // than one nic.
    if (primary_nics > 1) {
        callback(new Error('payload specifies more than 1 primary NIC'));
        return;
    }

    if (payload.hasOwnProperty('vga')
        && VM.VGA_TYPES.indexOf(payload.vga) === -1) {

        callback(new Error('Invalid VGA type: "' + payload.vga
            + '", supported types are: ' + VM.VGA_TYPES.join(',')));
        return;
    }

    // Ensure password is not too long
    if (payload.hasOwnProperty('vnc_password')
        && payload.vnc_password.length > 8) {

        callback(new Error('VNC password is too long, maximum length is 8 '
            + 'characters.'));
        return;
    }

    props = ['zfs_root_recsize', 'zfs_data_recsize'];
    for (prop in props) {
        prop = props[prop];
        if (payload.hasOwnProperty(prop)) {
            if (payload[prop] === 0 || payload[prop] === '') {
                // this is the default, so set it back to that.
                payload[prop] = 131072;
            } else if (!validRecordSize(payload[prop])) {
                callback(new Error('invalid ' + prop + ' (' + payload[prop]
                    + '), must be 512-131072 and a power of 2. '
                    + '(0 to disable)'));
                return;
            }
        }
    }
    props = ['zfs_root_compression', 'zfs_data_compression'];
    for (prop in props) {
        prop = props[prop];

        if (payload.hasOwnProperty(prop)) {
            if (VM.COMPRESSION_TYPES.indexOf(payload[prop]) === -1) {
                callback(new Error('invalid compression type for '
                    + payload[prop] + ', must be one of: '
                    + VM.COMPRESSION_TYPES.join(', ')));
            }
        }
    }

    // Ensure MACs and IPs are not already used on this vm
    // NOTE: can't check other nodes yet.

    lookupConflicts(macs, ips, vrids, function (error, conflict) {
        if (error) {
            callback(error);
        } else {
            if (conflict) {
                callback(new Error('Conflict detected with another '
                    + 'vm, please check the MAC, IP, and VRID'));
            } else {
                VM.log.debug('no conflicts');
                lookupInvalidNicTags(changed_nics, function (e) {
                    if (e) {
                        callback(e);
                    } else {
                        callback();
                    }
                });
            }
        }
    });
}

function createDelegatedDataset(payload, callback)
{
    var ds;
    var options;
    var zcfg = '';

    if (payload.delegate_dataset) {
        VM.log.info('creating delegated dataset.');
        if (!payload.hasOwnProperty('zfs_filesystem')) {
            callback(new Error('payload missing zfs_filesystem'));
            return;
        }
        ds = path.join(payload.zfs_filesystem, '/data');

        options = ['create'];
        if (payload.hasOwnProperty('zfs_data_compression')) {
            options.push('-o', 'compression=' + payload.zfs_data_compression);
        }
        if (payload.hasOwnProperty('zfs_data_recsize')) {
            options.push('-o', 'recsize=' + payload.zfs_data_recsize);
        }
        options.push(ds);

        zfs(options, function (err) {
            if (err) {
                callback(err);
                return;
            }

            zcfg = zcfg + 'add dataset; set name=' + ds + '; end\n';
            zonecfg(['-u', payload.uuid, zcfg], function (e, fds) {
                if (e) {
                    VM.log.error({'err': e, 'stderr': fds.stderr},
                        'unable to add delegated dataset to ' + payload.uuid);
                    callback(e);
                } else {
                    callback();
                }
            });
        });
    } else {
        callback();
    }
}

function buildAddRemoveList(vmobj, payload, type, key, updatable)
{
    var add = [];
    var add_key;
    var field;
    var newobj;
    var oldobj;
    var plural = type + 's';
    var remove = [];
    var remove_key;
    var update_key;

    // initialize some plurals
    add_key = 'add_' + plural;
    remove_key = 'remove_' + plural;
    update_key = 'update_' + plural;

    // There's no way to update properties on a disk or nic with zonecfg
    // currently.  Yes, really.  So any disks/nics that should be updated, we
    // remove then add with the new properties.
    if (payload.hasOwnProperty(update_key)) {
        for (newobj in payload[update_key]) {
            newobj = payload[update_key][newobj];
            for (oldobj in vmobj[plural]) {
                oldobj = vmobj[plural][oldobj];

                if (oldobj[key] === newobj[key]) {
                    // This is the one to update: remove and add.
                    remove.push(oldobj[key]);

                    // only some fields make sense to update.
                    for (field in updatable) {
                        field = updatable[field];
                        if (newobj.hasOwnProperty(field)) {
                            oldobj[field] = newobj[field];
                        }
                    }

                    add.push(oldobj);
                }
            }
        }
    }

    if (payload.hasOwnProperty(remove_key)) {
        for (newobj in payload[remove_key]) {
            newobj = payload[remove_key][newobj];
            remove.push(newobj);
        }
    }

    if (payload.hasOwnProperty(add_key)) {
        for (newobj in payload[add_key]) {
            newobj = payload[add_key][newobj];
            add.push(newobj);
        }
    }

    return ({'add': add, 'remove': remove});
}

function buildDiskZonecfg(vmobj, payload)
{
    var add = [];
    var disk;
    var lists;
    var remove = [];
    var zcfg = '';

    lists = buildAddRemoveList(vmobj, payload, 'disk', 'path',
        UPDATABLE_DISK_PROPS);
    remove = lists.remove;
    add = lists.add;

    // remove is a list of disk paths, add a remove for each now.
    for (disk in remove) {
        disk = remove[disk];
        zcfg = zcfg + 'remove device match=' + disk + '\n';
    }

    for (disk in add) {
        disk = add[disk];

        zcfg = zcfg + 'add device\n'
            + 'set match=' + disk.path + '\n'
            + 'add property (name=boot, value="'
            + (disk.boot ? 'true' : 'false') + '")\n'
            + 'add property (name=model, value="' + disk.model + '")\n';

        if (disk.hasOwnProperty('media')) {
            zcfg = zcfg
                + 'add property (name=media, value="'
                + disk.media + '")\n';
        }

        if (disk.hasOwnProperty('image_size')) {
            zcfg = zcfg
                + 'add property (name=image-size, value="'
                + disk.image_size + '")\n';
        } else if (disk.hasOwnProperty('size')) {
            zcfg = zcfg + 'add property (name=size, value="'
                + disk.size + '")\n';
        }

        if (disk.hasOwnProperty('image_uuid')) {
            zcfg = zcfg
                + 'add property (name=image-uuid, value="'
                + disk.image_uuid + '")\n';
        }

        if (disk.hasOwnProperty('image_name')) {
            zcfg = zcfg + 'add property (name=image-name, value="'
                + disk.image_name + '")\n';
        }

        zcfg = zcfg + 'end\n';
    }

    return zcfg;
}

function buildNicZonecfg(vmobj, payload)
{
    var add;
    var lists;
    var matches;
    var n;
    var new_primary;
    var nic;
    var nic_idx = 0;
    var remove;
    var updated_primary;
    var used_nic_indexes = [];
    var zcfg = '';

    if (vmobj.hasOwnProperty('nics')) {
        // check whether we're adding or updating to set the primary flag. If we
        // are also find the existing NIC with the primary flag. If that's not
        // being removed, update it to remove the primary flag.
        if (payload.hasOwnProperty('add_nics')) {
            for (nic in payload.add_nics) {
                nic = payload.add_nics[nic];
                if (nic.hasOwnProperty('primary')) {
                    new_primary = nic.mac;
                }
            }
        }
        if (payload.hasOwnProperty('update_nics')) {
            for (nic in payload.update_nics) {
                nic = payload.update_nics[nic];
                if (nic.hasOwnProperty('primary')) {
                    new_primary = nic.mac;
                }
            }
        }
        if (new_primary) {
            // find old primary
            for (nic in vmobj.nics) {
                nic = vmobj.nics[nic];
                if (nic.hasOwnProperty('primary') && nic.mac !== new_primary) {
                    // we have a new primary, so un-primary the old.
                    if (payload.hasOwnProperty('remove_nics')
                        && payload.remove_nics.indexOf(nic.mac) !== -1) {

                        // we're removing the old primary so: done.
                        break;
                    } else if (payload.hasOwnProperty('update_nics')) {
                        updated_primary = false;
                        for (n in payload.update_nics) {
                            n = payload.update_nics[n];
                            if (n.mac === nic.mac) {
                                n.primary = false;
                                updated_primary = true;
                            }
                        }
                        if (!updated_primary) {
                            payload.update_nics.push({'mac': nic.mac,
                                'primary': false});
                        }
                    } else {
                        // just add a new update to unset the
                        payload.update_nics =
                            [ {'mac': nic.mac, 'primary': false} ];
                    }
                }
            }
        }
    }

    lists = buildAddRemoveList(vmobj, payload, 'nic', 'mac',
        UPDATABLE_NIC_PROPS);
    remove = lists.remove;
    add = lists.add;

    // create a list of used indexes so we can find the free ones
    if (vmobj.hasOwnProperty('nics')) {
        for (n in vmobj.nics) {
            if (vmobj.nics[n].hasOwnProperty('interface')) {
                matches = vmobj.nics[n].interface.match(/^net(\d+)$/);
                if (matches) {
                    used_nic_indexes.push(Number(matches[1]));
                }
            }
        }
    }

    // assign next available interface for nics without one
    for (nic in add) {
        nic = add[nic];
        if (!nic.hasOwnProperty('interface')) {
            while (used_nic_indexes.indexOf(nic_idx) !== -1) {
                nic_idx++;
            }
            nic.interface = 'net' + nic_idx;
            used_nic_indexes.push(Number(nic_idx));
        }

        // Changing the VRID changes the MAC address too, since the VRID is
        // encoded in the MAC. This can't be done until after
        // buildAddRemoveList above, since mac is used as the key to figure
        // out which nic is which
        if (nic.hasOwnProperty('vrrp_vrid')) {
            nic.mac = vrrpMAC(nic.vrrp_vrid);
        }
    }

    // remove is a list of nic macs, add a remove for each now.
    for (nic in remove) {
        nic = remove[nic];
        zcfg = zcfg + 'remove net mac-addr=' + ruinMac(nic) + '\n';
    }

    // properties that don't require any validation - add them if they're
    // present:
    var nicProperties = ['ip', 'netmask', 'model', 'dhcp_server',
        'allow_dhcp_spoofing', 'blocked_outgoing_ports', 'allow_ip_spoofing',
        'allow_mac_spoofing', 'allow_restricted_traffic',
        'allow_unfiltered_promisc', 'vrrp_vrid', 'vrrp_primary_ip'];

    for (nic in add) {
        nic = add[nic];

        zcfg = zcfg
            + 'add net\n'
            + 'set physical=' + nic.interface + '\n'
            + 'set mac-addr=' + ruinMac(nic.mac) + '\n';

        if (nic.hasOwnProperty('nic_tag')) {
            zcfg = zcfg + 'set global-nic=' + nic.nic_tag + '\n';
        }

        if (nic.hasOwnProperty('gateway') && nic.gateway.length > 0) {
            zcfg = zcfg + 'add property (name=gateway, value="'
                + nic.gateway + '")\n';
        }

        if (nic.hasOwnProperty('primary') && nic.primary) {
            zcfg = zcfg + 'add property (name=primary, value="true")\n';
        }

        if (nic.hasOwnProperty('vlan_id') && (nic.vlan_id !== '0')) {
            zcfg = zcfg + 'set vlan-id=' + nic.vlan_id + '\n';
        }

        if (nic.hasOwnProperty('blocked_outgoing_ports')) {
            zcfg = zcfg
                + 'add property (name=blocked-outgoing-ports, value="'
                + nic.blocked_outgoing_ports.join(',') + '")\n';
        }

        for (var prop in nicProperties) {
            prop = nicProperties[prop];
            if (nic.hasOwnProperty(prop)) {
                zcfg = zcfg + 'add property (name=' + prop + ', value="'
                    + nic[prop] + '")\n';
            }
        }

        zcfg = zcfg + 'end\n';
    }

    return zcfg;
}

function buildFilesystemZonecfg(vmobj, payload)
{
    var add = [];
    var filesystem;
    var lists;
    var opt;
    var remove = [];
    var zcfg = '';

    lists = buildAddRemoveList(vmobj, payload, 'filesystem', 'target', []);
    remove = lists.remove;
    add = lists.add;

    // remove is a list of disk paths, add a remove for each now.
    for (filesystem in remove) {
        filesystem = remove[filesystem];
        zcfg = zcfg + 'remove fs match=' + filesystem + '\n';
    }

    for (filesystem in add) {
        filesystem = add[filesystem];

        zcfg = zcfg + 'add fs\n' + 'set dir=' + filesystem.target + '\n'
            + 'set special=' + filesystem.source + '\n' + 'set type='
            + filesystem.type + '\n';
        if (filesystem.hasOwnProperty('raw')) {
            zcfg = zcfg + 'set raw=' + filesystem.raw + '\n';
        }
        if (filesystem.hasOwnProperty('options')) {
            for (opt in filesystem.options) {
                opt = filesystem.options[opt];
                zcfg = zcfg + 'add options "' + opt + '"\n';
            }
        }
        zcfg = zcfg + 'end\n';
    }

    return zcfg;
}

function buildZonecfgUpdate(vmobj, payload)
{
    var brand;
    var tmp;
    var zcfg = '';

    VM.log.debug('vmobj:\n' + JSON.stringify(vmobj, null, 2));
    VM.log.debug('payload:\n' + JSON.stringify(payload, null, 2));

    if (vmobj && vmobj.hasOwnProperty('brand')) {
        brand = vmobj.brand;
    } else {
        brand = payload.brand;
    }

    // Global properties can just be set, no need to clear anything first.
    if (payload.hasOwnProperty('cpu_shares')) {
        zcfg = zcfg + 'set cpu-shares=' + payload.cpu_shares.toString() + '\n';
    }
    if (payload.hasOwnProperty('zfs_io_priority')) {
        zcfg = zcfg + 'set zfs-io-priority='
            + payload.zfs_io_priority.toString() + '\n';
    }
    if (payload.hasOwnProperty('max_lwps')) {
        zcfg = zcfg + 'set max-lwps=' + payload.max_lwps.toString() + '\n';
    }
    if (payload.hasOwnProperty('limit_priv')) {
        zcfg = zcfg + 'set limitpriv="' + payload.limit_priv + '"\n';
    }

    if (!BRAND_OPTIONS[brand].features.use_vm_autoboot
        && payload.hasOwnProperty('autoboot')) {

        // kvm autoboot is managed by the vm-autoboot attr instead
        zcfg = zcfg + 'set autoboot=' + payload.autoboot.toString() + '\n';
    }

    // Capped Memory properties are special
    if (payload.hasOwnProperty('max_physical_memory')
        || payload.hasOwnProperty('max_locked_memory')
        || payload.hasOwnProperty('max_swap')) {

        // Capped memory parameters need either an add or select first.
        if (vmobj.hasOwnProperty('max_physical_memory')
            || vmobj.hasOwnProperty('max_locked_memory')
            || vmobj.hasOwnProperty('max_swap')) {

            // there's already a capped-memory section, use that.
            zcfg = zcfg + 'select capped-memory; ';
        } else {
            zcfg = zcfg + 'add capped-memory; ';
        }

        if (payload.hasOwnProperty('max_physical_memory')) {
            zcfg = zcfg + 'set physical='
                + payload.max_physical_memory.toString() + 'm; ';
        }
        if (payload.hasOwnProperty('max_locked_memory')) {
            zcfg = zcfg + 'set locked='
                + payload.max_locked_memory.toString() + 'm; ';
        }
        if (payload.hasOwnProperty('max_swap')) {
            zcfg = zcfg + 'set swap='
                + payload.max_swap.toString() + 'm; ';
        }

        zcfg = zcfg + 'end\n';
    }

    // Capped CPU is special
    if (payload.hasOwnProperty('cpu_cap')) {
        if (vmobj.hasOwnProperty('cpu_cap')) {
            zcfg = zcfg + 'select capped-cpu; ';
        } else {
            zcfg = zcfg + 'add capped-cpu; ';
        }

        zcfg = zcfg + 'set ncpus='
            + (Number(payload.cpu_cap) * 0.01).toString() + '; end\n';
    }

    // set to empty string so property is removed when not true
    if (payload.hasOwnProperty('do_not_inventory')) {
        if (payload.do_not_inventory !== 'true') {
            payload.do_not_inventory = '';
        }
    }

    // Attributes
    function setAttr(attr, attr_name, value) {
        if (!value) {
            value = payload[attr_name];
        }

        if (payload.hasOwnProperty(attr_name)) {
            if ((typeof (value) !== 'boolean')
                && (!value || trim(value.toString()) === '')) {

                // empty values we either remove or ignore.
                if (vmobj.hasOwnProperty(attr_name)) {
                    zcfg = zcfg + 'remove attr name=' + attr + ';';
                    // else do nothing, we don't add empty values.
                }
            } else {
                if (vmobj.hasOwnProperty(attr_name)) {
                    zcfg = zcfg + 'select attr name=' + attr + '; ';
                } else {
                    zcfg = zcfg + 'add attr; set name="' + attr + '"; '
                        + 'set type=string; ';
                }
                zcfg = zcfg + 'set value="' + value.toString() + '"; end\n';
            }
        }
    }
    setAttr('billing-id', 'billing_id');
    setAttr('owner-uuid', 'owner_uuid');
    setAttr('package-name', 'package_name');
    setAttr('package-version', 'package_version');
    setAttr('tmpfs', 'tmpfs');
    setAttr('hostname', 'hostname');
    setAttr('dns-domain', 'dns_domain');
    setAttr('default-gateway', 'default_gateway');
    setAttr('do-not-inventory', 'do_not_inventory');

    if (payload.hasOwnProperty('resolvers')) {
        setAttr('resolvers', 'resolvers', payload.resolvers.join(','));
    }
    if (payload.hasOwnProperty('alias')) {
        tmp = '';
        if (payload.alias) {
            tmp = new Buffer(payload.alias).toString('base64');
        }
        setAttr('alias', 'alias', tmp);
    }

    if (BRAND_OPTIONS[brand].features.use_vm_autoboot) {
        setAttr('vm-autoboot', 'autoboot');
    }

    // XXX Used on KVM but can be passed in for 'OS' too. We only setAttr on KVM
    if (BRAND_OPTIONS[brand].features.type === 'KVM') {
        setAttr('ram', 'ram');
    }

    // NOTE: Thanks to normalizePayload() we'll only have these when relevant
    setAttr('vcpus', 'vcpus');
    setAttr('boot', 'boot');
    setAttr('cpu-type', 'cpu_type');
    setAttr('vga', 'vga');
    setAttr('vnc-port', 'vnc_port');
    setAttr('spice-port', 'spice_port');
    setAttr('virtio-txtimer', 'virtio_txtimer');
    setAttr('virtio-txburst', 'virtio_txburst');

    // We use base64 here for these next five options:
    //
    //  vnc_password
    //  spice_password
    //  spice_opts
    //  qemu_opts
    //  qemu_extra_opts
    //
    // since these can contain characters zonecfg doesn't like.
    //
    if (payload.hasOwnProperty('vnc_password')) {
        if (payload.vnc_password === ''
            && (vmobj.hasOwnProperty('vnc_password')
            && vmobj.vnc_password !== '')) {

            VM.log.warn('Warning: VNC password was removed for VM '
                + vmobj.uuid + ' but VM needs to be restarted for change to'
                + 'take effect.');
        }
        if (payload.vnc_password.length > 0
            && !vmobj.hasOwnProperty('vnc_password')) {

            VM.log.warn('Warning: VNC password was added to VM '
                + vmobj.uuid + ' but VM needs to be restarted for change to'
                + 'take effect.');
        }

        setAttr('vnc-password', 'vnc_password',
            new Buffer(payload.vnc_password).toString('base64'));
    }
    if (payload.hasOwnProperty('spice_password')) {
        if (payload.spice_password === ''
            && (vmobj.hasOwnProperty('spice_password')
            && vmobj.spice_password !== '')) {

            VM.log.warn('Warning: SPICE password was removed for VM '
                + vmobj.uuid + ' but VM needs to be restarted for change to'
                + 'take effect.');
        }
        if (payload.spice_password.length > 0
            && !vmobj.hasOwnProperty('spice_password')) {

            VM.log.warn('Warning: SPICE password was added to VM '
                + vmobj.uuid + ' but VM needs to be restarted for change to'
                + 'take effect.');
        }

        setAttr('spice-password', 'spice_password',
            new Buffer(payload.spice_password).toString('base64'));
    }
    if (payload.hasOwnProperty('spice_opts')) {
        setAttr('spice-opts', 'spice_opts',
            new Buffer(payload.spice_opts).toString('base64'));
    }
    if (payload.hasOwnProperty('qemu_opts')) {
        setAttr('qemu-opts', 'qemu_opts',
            new Buffer(payload.qemu_opts).toString('base64'));
    }
    if (payload.hasOwnProperty('qemu_extra_opts')) {
        setAttr('qemu-extra-opts', 'qemu_extra_opts',
            new Buffer(payload.qemu_extra_opts).toString('base64'));
    }

    // Handle disks
    if (payload.hasOwnProperty('disks')
        || payload.hasOwnProperty('add_disks')
        || payload.hasOwnProperty('update_disks')
        || payload.hasOwnProperty('remove_disks')) {

        zcfg = zcfg + buildDiskZonecfg(vmobj, payload);
    }

    if (payload.hasOwnProperty('fs_allowed')) {
        if (payload.fs_allowed === '') {
            zcfg = zcfg + 'clear fs-allowed\n';
        } else {
            zcfg = zcfg + 'set fs-allowed="' + payload.fs_allowed + '"\n';
        }
    }

    if (payload.hasOwnProperty('filesystems')
        || payload.hasOwnProperty('add_filesystems')
        || payload.hasOwnProperty('update_filesystems')
        || payload.hasOwnProperty('add_filesystems')) {

        zcfg = zcfg + buildFilesystemZonecfg(vmobj, payload);
    }

    zcfg = zcfg + buildNicZonecfg(vmobj, payload);

    return zcfg;
}

// Checks that QMP is responding to query-status and if so passes the boolean
// value of the hwsetup parameter to the callback.
function checkHWSetup(vmobj, callback)
{
    var q;
    var socket;

    q = new Qmp(VM.log);
    socket = vmobj.zonepath + '/root/tmp/vm.qmp';

    q.connect(socket, function (error) {
        if (error) {
            VM.log.error(error, 'q.connect(): Error: ' + error.message);
            callback(error);
            return;
        }
        q.command('query-status', null, function (e, result) {
            if (e) {
                VM.log.error(e, 'q.command(query-status): Error: ' + e.message);
                callback(e);
                return;
            }
            q.disconnect();
            callback(null, result.hwsetup ? true : false);
            return;
        });
    });
}

// cb (if set) will be called with an Error if we can't setup the interval loop
// otherwise when the loop is shut down.
function markProvisionedWhenHWSetup(vmobj, options, cb)
{
    var loop_interval = 3; // seconds
    var ival_handle;

    if (!cb) {
        // if no cb() is passed in just log if there was an error
        cb = function (err) {
            if (err) {
                VM.log.warn(err, 'cb() not passed, ignoring error: '
                    + err.message);
            }
        };
    }

    if (!BRAND_OPTIONS[vmobj.brand].features.wait_for_hwsetup) {
        // do nothing for zones where we don't wait for hwsetup
        cb(new Error('brand ' + vmobj.brand + ' does not support hwsetup'));
        return;
    }

    if (!options) {
        options = {};
    }

    // if caller wants they can change the interval
    if (options.hasOwnProperty('interval')) {
        loop_interval = options.interval;
    }

    ival_handle = setInterval(function () {
        VM.load(vmobj.uuid, function (err, obj) {
            var timeout_remaining;

            function done() {
                if (ival_handle) {
                    clearInterval(ival_handle);
                    ival_handle = null;
                }
            }

            if (err) {
                // If the VM was deleted between calls, nothing much we can do.
                VM.log.error(err, 'Unable to load ' + vmobj.uuid + ' '
                    + err.message);
                cb(err);
                done();
                return;
            }

            // we only do anything if we're still waiting for provisioning
            if (vmobj.state !== 'provisioning') {
                done();
                cb();
                return;
            }

            timeout_remaining =
                (Number(obj.transition_expire) - Date.now(0)) / 1000;

            if (timeout_remaining <= 0) {
                // IMPORTANT: this may run multiple times, must be idempotent

                markVMFailure(vmobj, function (mark_err) {
                    VM.log.warn(mark_err, 'zoneinit failed, zone is '
                        + 'being stopped for manual investigation.');
                    cb();
                    done();
                });
                return;
            }

            checkHWSetup(vmobj, function (check_err, result) {
                if (check_err) {
                    VM.log.debug(check_err, 'checkHWSetup Error: '
                        + check_err.message);
                    return;
                }

                if (result) {
                    VM.log.debug('QMP says VM ' + vmobj.uuid
                        + ' completed hwsetup');
                    unsetTransition(vmobj, function (unset_err) {
                        var provisioning;
                        var provision_success;

                        provisioning = path.join(vmobj.zonepath,
                            '/root/var/svc/provisioning');
                        provision_success = path.join(vmobj.zonepath,
                            '/root/var/svc/provision_success');

                        if (unset_err) {
                            VM.log.error(unset_err);
                        } else {
                            VM.log.debug('cleared transition to provisioning on'
                                + ' ' + vmobj.uuid);
                        }

                        fs.rename(provisioning, provision_success,
                            function (e) {

                            if (e) {
                                if (e.code !== 'ENOENT') {
                                    VM.log.error(e);
                                    cb(err);
                                } else {
                                    VM.log.debug(e);
                                    // in this case the file doesn't exist so
                                    // either moved it already from outside or
                                    // deleted the machine, either way the next
                                    // run of the Interval loop should fix.
                                }
                                return;
                            }

                            cb();
                            done();
                        });
                    });
                }
            });
        });
    }, loop_interval * 1000);
}

// m argument should have zonename and uuid members.
function markVMFailure(m, cb)
{
    if (!m || !m.hasOwnProperty('uuid') || !m.hasOwnProperty('zonename')) {
        cb(new Error('markVMFailure needs uuid + zonename'));
        return;
    }

    execFile('/usr/bin/ptree', ['-z', m.zonename],
        function (ptree_err, stdout, stderr) {
            var zcfg;

            if (ptree_err) {
                VM.log.error(ptree_err, 'unable to get ptree from ' + m.uuid
                    + ': ' + stderr);
            } else {
                VM.log.warn('processes running in ' + m.uuid
                    + ' at fail time:\n' + stdout);
            }
            VM.log.warn(ptree_err, 'zone setup failed, zone is being stopped '
                + 'for manual investigation.');

            // Mark the zone as 'failed'
            zcfg = 'add attr; set name=failed; set value="provisioning"; '
                + 'set type=string; end';

            zonecfg(['-u', m.uuid, zcfg], function (zonecfg_err) {

                if (zonecfg_err) {
                    VM.log.error(zonecfg_err, 'Unable to set failure flag on '
                        + m.uuid + ': ' + zonecfg_err.message);
                } else {
                    VM.log.debug('set failure flag on ' + m.uuid);
                }

                // attempt to remove transition
                unsetTransition(m, function (unset_err) {
                    if (unset_err) {
                        VM.log.error(unset_err);
                    }

                    VM.stop(m.uuid, {'force': true}, function (stop_err) {
                        // only log errors because there's nothing to do

                        if (stop_err) {
                            VM.log.error(stop_err, 'failed to stop VM ' + m.uuid
                                + ': ' + stop_err.message);
                        }

                        cb();
                    });
                });
            });
        }
    );
}

function waitForZoneinit(payload, cb)
{
    VM.waitForZoneState(payload, 'running', {}, function (error) {
        if (error) {
            markVMFailure(payload, function (err) {
                VM.log.warn(err, 'zoneinit failed, zone is being '
                    + 'stopped for manual investigation.');
                cb(err);
            });
        } else {
            // we're not still provisioning
            unsetTransition(payload, function (err) {
                cb(err);
            });
        }
    });
}

function svccfg(zonepath, args, callback)
{
    var cmd = '/usr/sbin/svccfg';
    var options = {};

    options = {
        env: {
            'SVCCFG_CONFIGD_PATH': '/lib/svc/bin/svc.configd',
            'SVCCFG_REPOSITORY':
                path.join(zonepath, 'root', '/etc/svc/repository.db')
        }
    };

    VM.log.debug({'command': cmd + ' ' + args.join(' '), 'options': options},
        'modifying svc repo in ' + zonepath);
    execFile(cmd, args, options, function (error, stdout, stderr) {
        if (error) {
            callback(error, {'stdout': stdout, 'stderr': stderr});
        } else {
            callback(null, {'stdout': stdout, 'stderr': stderr});
        }
    });
}


// This calls cb() when /var/svc/provisioning is gone. When this calls cb()
// with an Error object, the provision is considered failed so this should
// only happen when something timed out that is unrelated to the user.
//
// IMPORTANT: this is only intended to be used by vmadmd. Do not use elsewhere!
exports.waitForProvisioning = function (vmobj, cb)
{
    var dirname = path.join(vmobj.zonepath, 'root', '/var/svc');
    var filename = path.join(dirname, 'provisioning');
    var timeout;
    var timeout_remaining = PROVISION_TIMEOUT; // default to whole thing
    var watcher;

    function done() {
        if (timeout) {
            VM.log.debug('clearing timeout for ' + vmobj.uuid);
            clearTimeout(timeout);
            timeout = null;
        }
        if (watcher) {
            VM.log.debug('closing watcher for ' + vmobj.uuid);
            watcher.close();
            watcher = null;
        }
    }

    if ((vmobj.state === 'provisioning')
        && (vmobj.hasOwnProperty('transition_expire'))) {

        timeout_remaining =
            (Number(vmobj.transition_expire) - Date.now(0)) / 1000;

        // Always give it at least 1 second's chance.
        if (timeout_remaining < 1) {
            timeout_remaining = 1;
        }
    } else {
        // don't know what to do here we're not provisioning.
        VM.log.warn('waitForProvisioning called when ' + vmobj.uuid
            + ' was not provisioning');
        cb();
        return;
    }

    VM.log.debug({
        'transition_expire': Number(vmobj.transition_expire),
        'now': Date.now(0)
    }, 'waiting ' + timeout_remaining + ' sec(s) for provisioning');

    timeout = setTimeout(function () {
        markVMFailure(vmobj, function (err) {
            var errstr = 'timed out waiting for /var/svc/provisioning to move';
            if (err) {
                VM.log.warn(err, 'markVMFailure(): ' + err.message);
            }
            cb(new Error(errstr));
            VM.log.error(errstr);
            done();
        });
    }, (timeout_remaining * 1000));

    VM.log.debug('created timeout for ' + vmobj.uuid);

    // this starts a loop that will move provisioning -> provision_success when
    // the hardware of the VM has been initialized the first time.
    if (BRAND_OPTIONS[vmobj.brand].features.wait_for_hwsetup) {
        markProvisionedWhenHWSetup(vmobj);
    }

    watcher = fs.watch(filename, function (evt, file) {
        // We only care about 'rename' which also fires when the file is
        // deleted.
        VM.log.debug('watcher.event(' + vmobj.uuid + '): ' + evt);
        if (evt === 'rename') {
            fs.exists(filename, function (exists) {
                if (exists) {
                    // somehow we still have /var/svc/provisioning!
                    markVMFailure(vmobj, function (err) {
                        if (err) {
                            VM.log.warn(err, 'markVMFailure(): ' + err.message);
                        }
                        cb(new Error('provisioning exists after rename!'));
                        done();
                    });
                    return;
                }

                // So long as /var/svc/provisioning is gone, we don't care what
                // replaced it.  Success or failure of user script doesn't
                // matter for the state, it's provisioned now.
                unsetTransition(vmobj, function (err) {
                    cb();
                    done();
                });
                return;
            });
        }
    });

    VM.log.debug('created watcher for ' + vmobj.uuid);

};

// create and install a 'joyent' or 'kvm' brand zone.
function installZone(payload, callback)
{
    var receiving = false;
    var var_svc_provisioning = false;
    var vmobj;
    var zoneinit = {};

    VM.log.debug('installZone()');

    async.series([
        function (cb) {
            VM.load(payload.uuid, function (err, obj) {
                if (err) {
                    cb(err);
                    return;
                }
                vmobj = obj;
                cb();
            });
        }, function (cb) {
            var thing;
            var missing = false;
            var msg;
            var things = ['datasets', 'filesystems', 'disks'];

            if (vmobj.state === 'receiving') {
                receiving = true;
                msg = 'zone is still missing:';
                for (thing in things) {
                    thing = things[thing];
                    if (vmobj.missing[thing].length !== 0) {
                        msg = msg + ' ' + vmobj.missing[thing].length + ' '
                            + thing + ',';
                        missing = true;
                    }
                }
                msg = rtrim(msg, ',');

                if (missing) {
                    cb(new Error('Unable to complete install for '
                        + vmobj.uuid + ' ' + msg));
                    return;
                }
            }
            cb();
        }, function (cb) {
            // Install the zone.
            // This will create the dataset and mark the zone 'installed'.
            var args = ['-z', vmobj.zonename, 'install', '-q',
                payload.quota.toString()];

            // For both OS and KVM VMs you can pass an image_uuid at the
            // top-level. This will be your zone's root dataset. On KVM the user
            // is never exposed to this. It's used there for something like
            // SPICE.
            if (payload.hasOwnProperty('image_uuid')) {
                args.push('-t', payload.image_uuid, '-x', 'nodataset');
            }

            zoneadm(args, function (err, fds) {
                if (err) {
                    VM.log.error({'err': err, 'stdout': fds.stdout,
                        'stderr': fds.stderr}, 'zoneadm failed to install: '
                        + err.message);
                    cb(err);
                } else {
                    cb();
                }
            });
        }, function (cb) {
            // Apply compression if set
            var options = [];
            if (payload.hasOwnProperty('zfs_root_compression')) {
                options = ['set', 'compression='
                    + payload.zfs_root_compression, payload.zfs_filesystem];
                zfs(options, function (err) {
                    cb(err);
                });
            } else {
                cb();
            }
        }, function (cb) {
            // Apply recsize if set
            var options = [];
            if (payload.hasOwnProperty('zfs_root_recsize')) {
                options = ['set', 'recsize=' + payload.zfs_root_recsize,
                    payload.zfs_filesystem];
                zfs(options, function (err) {
                    cb(err);
                });
            } else {
                cb();
            }
        }, function (cb) {
            // Some zones can have an additional 'data' dataset delegated to
            // them for use in the zone.  This will set that up.  If the option
            // is not set, the following does nothing.
            if (!receiving) {
                createDelegatedDataset(payload, function (err) {
                    if (err) {
                        cb(err);
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            // Write out the zone's metadata
            // Note: we don't do this when receiving because dataset will
            // already contain metadata and we don't want to wipe that out.
            if (!receiving) {
                saveMetadata(payload, function (err) {
                    if (err) {
                        VM.log.error(err, 'unable to save metadata: '
                            + err.message);
                        cb(err);
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            // if we were receiving, we're done receiving now
            if (receiving) {
                unsetTransition(vmobj, cb);
            } else {
                cb();
            }
        }, function (cb) {
            // load /var/zoneinit/zoneinit.json file if it's relevant:

            var filename;
            // var zoneinit is in installZone() scope

            // when receiving zoneinit is never run.
            if (receiving) {
                cb();
                return;
            }

            filename = path.join(vmobj.zonepath, 'root',
                '/var/zoneinit/zoneinit.json');

            if (BRAND_OPTIONS[vmobj.brand].features.zoneinit) {
                fs.readFile(filename, function (error, data) {
                    if (error && (error.code === 'ENOENT')) {
                        // doesn't exist, leave empty
                        VM.log.debug('zoneinit.json does not exist.');
                        cb();
                    } else if (error) {
                        // error reading: fail.
                        cb(error);
                    } else {
                        // success try to load json
                        try {
                            zoneinit = JSON.parse(data.toString());
                            VM.log.debug({'zoneinit_json': zoneinit},
                                'parsed zoneinit.json');
                        } catch (e) {
                            zoneinit = {};
                            VM.log.error(e);
                        }
                        cb();
                    }
                });
            } else {
                VM.log.debug('brand does not support '
                    + '/var/zoneinit/zoneinit.json, not trying to load.');
                cb();
            }
        }, function (cb) {
            // For joyent and joyent-minimal at least, set the timeout for the
            // svc start method to the value specified in the payload, or a
            // default.

            var timeout;

            if (BRAND_OPTIONS[vmobj.brand].features.update_mdata_exec_timeout) {

                if (payload.hasOwnProperty('mdata_exec_timeout')) {
                    timeout = payload.mdata_exec_timeout;
                } else {
                    timeout = DEFAULT_MDATA_TIMEOUT;
                }

                svccfg(vmobj.zonepath, [
                    '-s', 'svc:/smartdc/mdata:execute',
                    'setprop', 'start/timeout_seconds', '=', 'count:', timeout
                    ], function (error, stdio) {

                    if (error) {
                        VM.log.error(error, 'failed to set mdata:exec timeout');
                        cb(error);
                        return;
                    }

                    cb();
                });
            } else {
                cb();
            }

        }, function (cb) {

            var filename;
            // var_svc_provisioning is at installZone() scope

            // If we're not receiving, we're provisioning a new VM and in that
            // case we write the /var/svc/provisioning file which should exist
            // until something in the zone decides provisioning is complete. At
            // that point it will be moved to either:
            //
            //    /var/svc/provisioning_success
            //    /var/svc/provisioning_failure
            //
            // to indicate that the provisioning setup has been completed.

            if (receiving) {
                cb();
                return;
            }

            if (BRAND_OPTIONS[vmobj.brand].features.var_svc_provisioning) {
                // these brands always handle /var/svc/provisioning on this
                // platform.

                var_svc_provisioning = true;
            } else {
                if (zoneinit.hasOwnProperty('features')) {
                    if (zoneinit.features.var_svc_provisioning) {
                        VM.log.debug('features.var_svc_provisioning == true');
                        var_svc_provisioning = true;
                    }
                } else {
                    // didn't load zoneinit features, so check for datasets that
                    // have 04-mdata.sh
                    filename = path.join(vmobj.zonepath,
                        '/root/root/zoneinit.d/04-mdata.sh');
                    if (fs.existsSync(filename)) {
                        VM.log.debug('/root/zoneinit.d/04-mdata.sh exists');
                        var_svc_provisioning = true;
                    } else {
                        VM.log.debug('/root/zoneinit.d/04-mdata.sh does not '
                            + 'exist');
                    }
                }
            }

            if (var_svc_provisioning) {
                fs.writeFile(path.join(vmobj.zonepath, 'root',
                    '/var/svc/provisioning'), '', function (err, result) {

                    if (err) {
                        VM.log.error(err, 'failed to create '
                            + '/var/svc/provisioning: ' + err.message);
                    } else {
                        VM.log.debug('created /var/svc/provisioning in '
                            + path.join(vmobj.zonepath, 'root'));
                    }

                    cb(err);
                });
            } else {
                VM.log.debug('VM does not support /var/svc/provisioning, '
                    + 'not creating');
                cb();
            }
        }, function (cb) {
            // This writes out the 'zoneconfig' file used by zoneinit to root's
            // home directory in the zone.
            if (! receiving
                && BRAND_OPTIONS[vmobj.brand].features.zoneinit
                && (! zoneinit.hasOwnProperty('features')
                || zoneinit.features.zoneconfig)) {

                // No 'features' means old dataset.  If we have old dataset or
                // one that really wants a zoneconfig, write it out.

                writeZoneconfig(payload, function (err) {
                    cb(err);
                });
            } else {
                cb();
            }
        }, function (cb) {
            if (BRAND_OPTIONS[vmobj.brand].features.write_zone_netfiles
                && !receiving) {

                writeZoneNetfiles(payload, function (err) {
                    cb(err);
                });
            } else {
                cb();
            }
        }, function (cb) {
            if (vmobj.hasOwnProperty('zonepath')
                && BRAND_OPTIONS[vmobj.brand].features.cleanup_dataset
                && !receiving) {

                cleanupMessyDataset(vmobj.zonepath, vmobj.brand,
                    function (err) {

                    cb(err);
                });
            } else {
                cb();
            }
        }, function (cb) {
            // The vm is now ready to start, we'll start if autoboot is set.
            if (!payload.autoboot) {
                cb();
                return;
            }

            VM.start(payload.uuid, {}, function (err, res) {
                if (err) {
                    cb(err);
                } else {
                    cb();
                }
            });
        }, function (cb) {

            // var_svc_provisioning is at installZone() scope

            // In these cases we never wait for provisioning -> running
            if (!payload.autoboot || payload.nowait || receiving) {
                cb();
                return;
            }

            // most VMs support the /var/svc/provision{ing,_success,_failure}
            // files. For those, if !nowait, we wait for the file to change
            // from provisioning -> either provision_success, or
            // provision_failure.

            if (var_svc_provisioning && vmobj.state === 'provisioning') {
                // wait for /var/svc/provisioning -> provision_success/failure

                VM.waitForProvisioning(vmobj, function (err) {
                    VM.log.debug(err, 'waited for provisioning');

                    if (!err) {
                        VM.log.info('provisioning complete: '
                            + '/var/svc/provisioning is gone');
                        // this will clear the provision transition
                        unsetTransition(vmobj, function (unset_err) {
                            cb(unset_err);
                        });
                    } else {
                        // fail
                        VM.log.error(err);
                        cb(err);
                    }
                });
            } else {
                // won't have /var/svc/provision_success, so do things the old
                // way and just wait for the zone to go running (meaning it
                // rebooted after zoneinit).

                waitForZoneinit(payload, function (err) {
                    VM.log.info(err, 'waited for zoneinit');
                    cb(err);
                });
            }

        }], function (error) {
            callback(error);
        }
    );
}

// create and install a 'joyent' or 'kvm' brand zone.
function createZone(payload, callback)
{
    var create_time;
    var n;
    var now = new Date;
    var primary_found;
    var provision_timeout = PROVISION_TIMEOUT;
    var t;
    var zcfg;

    VM.log.debug('createZone()');

    payload.zfs_filesystem = payload.zpool + '/' + payload.zonename;
    payload.zonepath = '/' + payload.zfs_filesystem;

    // we add create-timestamp in all cases except where we're receiving since
    // in that case we want to preserve the original create-timestamp.
    if (!payload.hasOwnProperty('transition')
        || (payload.transition.transition !== 'receiving')
        || !payload.hasOwnProperty('create_timestamp')) {

        create_time = now.toISOString();
    } else {
        create_time = payload.create_timestamp;
    }

    // set the properties that can't be updated later here.
    zcfg = 'create -b\n'
        + 'set zonepath=' + payload.zonepath + '\n'
        + 'set brand=' + payload.brand + '\n'
        + 'set uuid=' + payload.uuid + '\n'
        + 'set ip-type=exclusive\n'
        + 'add attr; set name="create-timestamp"; set type=string; set value="'
        + create_time + '"; end\n';

    if (payload.hasOwnProperty('transition')) {
        // IMPORTANT: this is for internal use only and should not be documented
        // as an option for create's payload.  Used for receive.
        t = payload.transition;
        zcfg = zcfg
            + buildTransitionZonecfg(t.transition, t.target, t.timeout) + '\n';
    } else {
        // Assume this is really a new VM, add transition called 'provisioning'
        // only if the machine is going to be booting.
        if (!payload.hasOwnProperty('autoboot') || payload.autoboot) {
            zcfg = zcfg + buildTransitionZonecfg('provisioning', 'running',
                provision_timeout * 1000) + '\n';
        }
    }

    // We call the property 'dataset-uuid' even though the property name is
    // image_uuid because existing VMs in the wild will be using dataset-uuid
    // already, and we are the point where the image becomes a dataset anyway.
    if (payload.hasOwnProperty('image_uuid')) {
        zcfg = zcfg + 'add attr; set name="dataset-uuid"; set type=string; '
            + 'set value="' + payload.image_uuid + '"; end\n';
    }

    // if do_not_inventory is set, we don't don't want there to be a window
    // where the VM might get accidentally imported.
    if (payload.hasOwnProperty('do_not_inventory')
        && payload.do_not_inventory) {

        zcfg = zcfg + 'add attr; set name="do-not-inventory"; set type=string; '
            + 'set value="true"; end\n';
    }

    if (BRAND_OPTIONS[payload.brand].features.use_vm_autoboot) {
        // we always set autoboot=false for VM zones, since we want vmadmd to
        // boot them and not the zones tools.  Use vm-autoboot to control VMs
        zcfg = zcfg + 'set autoboot=false\n';
    }

    // ensure that we have a primary nic, even if one wasn't specified
    if (payload.hasOwnProperty('add_nics') && payload.add_nics.length != 0) {
        primary_found = false;

        for (n in payload.add_nics) {
            n = payload.add_nics[n];
            if (n.hasOwnProperty('primary') && n.primary) {
                primary_found = true;
                break;
            }
        }
        if (!primary_found) {
            payload.add_nics[0].primary = true;
        }
    }

    // Passing an empty first parameter here, tells buildZonecfgUpdate that
    // we're talking about a new machine.
    zcfg = zcfg + buildZonecfgUpdate({}, payload);

    // include the zonecfg in the debug output to help track down problems.
    VM.log.debug(zcfg);

    // send the zonecfg data we just generated as a file to zonecfg,
    // this will create the zone.
    zonecfgFile(zcfg, ['-z', payload.zonename], function (err, fds) {
        if (err || payload.create_only) {
            callback(err);
        } else {
            installZone(payload, callback);
        }
    });
}

function normalizeNics(payload, vmobj)
{
    var n;
    var nic;

    // ensure all NICs being created/added have a MAC, remove the 'index' if it
    // is passed (that's deprecated), rename 'interface' to 'physical'.
    if (payload.hasOwnProperty('add_nics')) {
        for (n in payload.add_nics) {
            if (payload.add_nics.hasOwnProperty(n)) {
                nic = payload.add_nics[n];

                if (!nic.hasOwnProperty('mac')
                    && !nic.hasOwnProperty('vrrp_vrid')) {
                    nic.mac = generateMAC();
                }
                delete nic.index;
                if (nic.hasOwnProperty('interface')) {
                    nic.physical = nic.interface;
                    delete nic.interface;
                }

                // nics.*.primary only supports true value, unset false. We also
                // handle the case here why they used the deprecated '1' value.
                // We will have already warned them, but still support for now.
                if (nic.hasOwnProperty('primary')) {
                    if (nic.primary || nic.primary === '1'
                        || nic.primary === 1) {

                        nic.primary = true;
                    } else {
                        delete nic.primary;
                    }
                }
            }
        }
    }
}

/*
 * This is called for both create and update, everything here should be safe for
 * both.  The vmobj will be set if it's an update.
 *
 */
function normalizePayload(payload, vmobj, callback)
{
    var action;
    var allowed;
    var brand;
    var property;

    // fix type of arguments that should be numbers, do this here so that fixing
    // memory works correctly later using math.
    for (property in payload) {
        if (payload.hasOwnProperty(property)) {
            if (PAYLOAD_PROPERTIES.hasOwnProperty(property)
                && PAYLOAD_PROPERTIES[property].type === 'integer'
                && payload[property] !== undefined) {
                // undefined is a special case since we use that to unset props

                payload[property] = Number(payload[property]);
                if (isNaN(payload[property])) {
                    callback(new Error('Invalid value for ' + property + ': '
                        + JSON.stringify(payload[property]) + ':'
                        + typeof (payload[property])));
                    return;
                }
            }
        }
    }

    if (payload.hasOwnProperty('quota') && payload.quota === undefined) {
        // when unsetting quota we set to 0
        payload.quota = 0;
    }

    if (vmobj) {
        /* update */
        fixPayloadMemory(payload, vmobj);
        action = 'update';
    } else {
        /* this also calls fixPayloadMemory() */
        applyZoneDefaults(payload);

        if (payload.hasOwnProperty('create_only')
            && payload.transition.transition === 'receiving') {

            action = 'receive';
        } else {
            action = 'create';
        }
    }

    // Should always have a brand after we applied defaults.
    if (vmobj && vmobj.hasOwnProperty('brand')) {
        brand = vmobj.brand;
    } else if (payload.hasOwnProperty('brand')) {
        brand = payload.brand;
    } else {
        callback(new Error('Unable to determine brand for payload'));
        return;
    }

    // Historically we supported dataset_uuid for joyent+joyent-minimal and
    // zone_dataset_uuid for kvm. Now we just support image_uuid so give a
    // deprecation warning and translate if old version specified. This needs
    // to happen before VM.validate because image_uuid is required for most
    // VMs.
    allowed = BRAND_OPTIONS[brand].allowed_properties;
    if ((allowed.hasOwnProperty('dataset_uuid')
            && payload.hasOwnProperty('dataset_uuid'))
        || (allowed.hasOwnProperty('zone_dataset_uuid')
            && payload.hasOwnProperty('zone_dataset_uuid'))) {

        property = (payload.hasOwnProperty('dataset_uuid') ? 'dataset_uuid'
            : 'zone_dataset_uuid');

        if (payload.hasOwnProperty('image_uuid')) {
            VM.log.warn('DEPRECATED option ' + property + ' found, '
                + 'ignoring. In the future use image_uuid only.');
        } else {
            VM.log.warn('DEPRECATED option ' + property + ' found, '
                + 'ignoring. In the future use image_uuid instead.');
            payload.image_uuid = payload[property];
            delete payload.dataset_uuid;
        }
    }

    // after ZoneDefaults have been applied, we should always have zone. Now
    // we validate the payload properties and remove any that are invalid. If
    // there are bad values we'll just fail.
    VM.validate(brand, action, payload, function (errors) {
        var bad_prop;
        var compound_props = ['disks', 'nics', 'filesystems'];
        var matches;
        var obj;
        var prop;

        if (errors) {
            if (errors.hasOwnProperty('bad_brand')) {
                callback(new Error('Invalid brand while validating payload: '
                    + JSON.stringify(brand)));
                return;
            }
            if (errors.bad_values.length > 0) {
                callback(new Error('Invalid value(s) for: '
                    + errors.bad_values.join(',')));
                return;
            }
            if (errors.missing_properties.length > 0) {
                callback(new Error('Missing required properties: '
                    + errors.missing_properties.join(',')));
                return;
            }
            for (bad_prop in errors.bad_properties) {
                bad_prop = errors.bad_properties[bad_prop];
                VM.log.warn('Warning, invalid ' + action + ' property: ['
                    + bad_prop + '] removing from payload.');

                // for bad properties like nics.*.allow_unfiltered_promisc we
                // need to remove it from add_nics, update_nics, etc.
                for (prop in compound_props) {
                    prop = compound_props[prop];

                    matches = new RegExp('^' + prop
                        + '\\.\\*\\.(.*)$').exec(bad_prop);
                    if (matches) {
                        if (payload.hasOwnProperty(prop)) {
                            for (obj in payload[prop]) {
                                delete payload[prop][obj][matches[1]];
                            }
                        }
                        if (payload.hasOwnProperty('add_' + prop)) {
                            for (obj in payload['add_' + prop]) {
                                delete payload['add_' + prop][obj][matches[1]];
                            }
                        }
                        if (payload.hasOwnProperty('update_' + prop)) {
                            for (obj in payload['update_' + prop]) {
                                delete payload['update_'
                                    + prop][obj][matches[1]];
                            }
                        }
                    }
                }

                delete payload[bad_prop];
            }
        }

        // By the time we got here all the properties in the payload are allowed

        // Now we make sure we've got a zonename (use uuid if not already set)
        // note that in some cases we don't have a UUID yet, so if we generate
        // one we will set zonename in createZoneUUID()
        if (!payload.hasOwnProperty('zonename')
            || payload.zonename === undefined) {

            payload.zonename = payload.uuid;
        }

        // You use 'disks' and 'nics' when creating, but the underlying
        // functions expect add_disks and add_nics, so we rename them now that
        // we've confirmed we've got the correct thing for this action.
        if (payload.hasOwnProperty('disks')) {
            if (payload.hasOwnProperty('add_disks')) {
                callback(new Error('Cannot specify both "disks" and '
                    + '"add_disks"'));
                return;
            }
            payload.add_disks = payload.disks;
            delete payload.disks;
        }
        if (payload.hasOwnProperty('nics')) {
            if (payload.hasOwnProperty('add_nics')) {
                callback(new Error('Cannot specify both "nics" and '
                    + '"add_nics"'));
                return;
            }
            payload.add_nics = payload.nics;
            delete payload.nics;
        }
        if (payload.hasOwnProperty('filesystems')) {
            if (payload.hasOwnProperty('add_filesystems')) {
                callback(new Error('Cannot specify both "filesystems" and '
                    + '"add_filesystems"'));
                return;
            }
            payload.add_filesystems = payload.filesystems;
            delete payload.filesystems;
        }

        // if there's a zfs_root_* and no zfs_data_*, normally the properties
        // would fall through, we don't want that.
        if (payload.hasOwnProperty('zfs_root_compression')
            && !payload.hasOwnProperty('zfs_data_compression')) {

            if (vmobj && vmobj.hasOwnProperty('zfs_data_compression')) {
                // keep existing value.
                payload.zfs_data_compression = vmobj.zfs_data_compression;
            } else {
                // keep default value.
                payload.zfs_data_compression = 'off';
            }
        }
        if (payload.hasOwnProperty('zfs_root_recsize')
            && !payload.hasOwnProperty('zfs_data_recsize')) {

            if (vmobj && vmobj.hasOwnProperty('zfs_data_recsize')) {
                // keep existing value.
                payload.zfs_data_recsize = vmobj.zfs_data_recsize;
            } else {
                // keep default value.
                payload.zfs_data_recsize = 131072;
            }
        }

        // this will ensure we've got a MAC, etc.
        normalizeNics(payload, vmobj);

        // Fix types for boolean fields in case someone put in 'false'/'true'
        // instead of false/true
        for (property in payload) {
            if (payload.hasOwnProperty(property)) {
                if (PAYLOAD_PROPERTIES.hasOwnProperty(property)
                    && PAYLOAD_PROPERTIES[property].type === 'boolean') {

                    payload[property] = fixBooleanLoose(payload[property]);
                }
            }
        }

        // We used to support zfs_storage_pool_name, but zpool is better.
        if (payload.hasOwnProperty('zfs_storage_pool_name')) {
            if (payload.hasOwnProperty('zpool')) {
                VM.log.warn('DEPRECATED option zfs_storage_pool_name found, '
                    + ' ignoring!');
            } else {
                VM.log.warn('DEPRECATED option zfs_storage_pool_name found, '
                    + ' replacing with zpool!');
                payload.zpool = payload.zfs_storage_pool_name;
                delete payload.zfs_storage_pool_name;
            }
        }

        // When creating a VM with SPICE you need the image_uuid, if you don't
        // pass that, we'll remove any SPICE options.
        if (action === 'create'
            && !payload.hasOwnProperty('image_uuid')) {

            if (payload.hasOwnProperty('spice_opts')
                || payload.hasOwnProperty('spice_password')
                || payload.hasOwnProperty('spice_port')) {

                VM.log.warn('Creating with SPICE options requires '
                    + 'image_uuid, REMOVING spice_*');
                delete payload.spice_opts;
                delete payload.spice_password;
                delete payload.spice_port;
            }
        }

        system.getProvisionableMemory(function (err, available_MiB) {
            if (err) {
                VM.log.warn('unable to determine system memory '
                    + 'usage, assuming we have sufficient memory.');
                payload.available_MiB = payload.ram;
            } else {
                payload.available_MiB = available_MiB;
            }
            checkPayloadProperties(payload, vmobj, function (e) {
                if (e) {
                    callback(e);
                } else {
                    callback();
                }
            });
        });
    });
}

function buildTransitionZonecfg(transition, target, timeout)
{
    var cmdline;

    cmdline = 'add attr; set name=transition; set value="'
        + transition + ':' + target + ':' + (Date.now(0) + timeout).toString()
        + '"; set type=string; end';

    return cmdline;
}

function unsetTransition(m, callback)
{
    zonecfg(['-u', m.uuid, 'remove attr name=transition'], function (err, fds) {
        if (err) {
            VM.log.error('unable to remove transition for zone '
                + m.uuid + ' stderr: ' + fds.stderr, err);
        }
        // We don't error here because we want to be idempotent.

        if (m.transition_to && m.transition_to === 'start') {
            VM.log.debug('vm was stopping for reboot, '
                + 'transitioning to start.');
            VM.start(m.uuid, {}, function (e) {
                if (e) {
                    VM.log.error('failed to start when clearing '
                        + 'transition', e);
                }
            });
        }

        callback();
    });
}

function setTransition(m, transition, target, timeout, callback)
{
    if (!timeout) {
        callback(new Error('setTransition() requires timeout argument.'));
        return;
    }

    async.series([
        function (cb) {
            if (m.hasOwnProperty('transition')) {
                unsetTransition(m, cb);
            } else {
                cb();
            }
        }, function (cb) {
            var zcfg;

            zcfg = buildTransitionZonecfg(transition, target, timeout);
            zonecfg(['-u', m.uuid, zcfg], function () {
                // TODO: handle errors

                VM.log.debug('setting transition=' + transition
                    + ' for vm ' + m.uuid);

                cb();
            });
        }
    ], function (error) {
        callback(error);
    });
}

function receiveVM(json, callback)
{
    var payload = {};

    try {
        payload = JSON.parse(json);
    } catch (e) {
        callback(e);
        return;
    }

    payload.create_only = true;

    // adding transition here is considered to be *internal only* not for
    // consumer use and not to be documented as a property you can use with
    // create.
    payload.transition =
        {'transition': 'receiving', 'target': 'stopped', 'timeout': 86400};

    // We delete tags and metadata here becasue this exists in the root
    // dataset which we will be copying, so it would be duplicated here.
    delete payload.customer_metadata;
    delete payload.internal_metadata;
    delete payload.tags;

    VM.create(payload, function (err, result) {
        if (err) {
            callback(err);
        }

        // don't include the special transition in the payload we write out.
        delete payload.transition;

        fs.writeFile('/etc/zones/' + payload.uuid + '-receiving.json',
            JSON.stringify(payload, null, 2), function (e) {

            if (e) {
                callback(e);
                return;
            }

            // ready for datasets
            callback(null, result);
        });
    });
}

function receiveStdinChunk(type, callback)
{
    var child;
    var chunk_name = '';
    var chunk_size = 0;
    var json = '';
    var remaining = '';

    /*
     * XXX
     *
     * node 0.6.x removed support for arbitrary file descriptors which
     * means we can only handle stdin for now since we need to pass this
     * descriptor directly to the child.  0.8.x is supposed to reintroduce
     * this functionality.  When we do, this should be changed to open
     * the file and set fd to the descriptor, and we should be able to
     * get rid of vmunbundle.
     *
     */

    if (type === 'JSON') {
        VM.log.info('/usr/vm/sbin/vmunbundle json');
        child = spawn('/usr/vm/sbin/vmunbundle', ['json'],
            {customFds: [0, -1, -1]});
    } else if (type === 'DATASET') {
        VM.log.info('/usr/vm/sbin/vmunbundle dataset');
        child = spawn('/usr/vm/sbin/vmunbundle', ['dataset'],
            {customFds: [0, -1, -1]});
    } else {
        callback(new Error('Unsupported chunk type ' + type));
    }

    child.stderr.on('data', function (data) {
        var idx;
        var line;
        var matches;

        remaining += data.toString();

        idx = remaining.indexOf('\n');
        while (idx > -1) {
            line = trim(remaining.substring(0, idx));
            remaining = remaining.substring(idx + 1);

            VM.log.debug('VMUNBUNDLE: ' + line);
            matches = line.match(/Size: ([\d]+)/);
            if (matches) {
                chunk_size = Number(matches[1]);
            }
            matches = line.match(/Name: \[(.*)\]/);
            if (matches) {
                chunk_name = matches[1];
            }

            idx = remaining.indexOf('\n');
        }
    });

    child.stdout.on('data', function (data) {
        json += data.toString();
        VM.log.debug('json size is ' + json.length);
    });

    child.on('close', function (code) {
        VM.log.debug('vmunbundle process exited with code ' + code);
        if (code === 3) {
            VM.log.debug('vmbundle: end of bundle.');
            callback(null, 'EOF');
            return;
        } else if (code !== 0) {
            callback(new Error('vmunbundle exited with code ' + code));
            return;
        }

        // if it was a dataset, we've now imported it.
        // if it was json, we've now got it in the json var.

        if (type === 'DATASET') {
            VM.log.info('Imported dataset ' + chunk_name);
            // delete 'sending' snapshot
            zfs(['destroy', '-F', chunk_name + '@sending'],
                function (err, fds) {
                    if (err) {
                        VM.log.warn(err, 'Failed to destroy ' + chunk_name
                            + '@sending: ' + err.message);
                    }
                    callback();
                }
            );
        } else if (type === 'JSON' && chunk_name === 'JSON'
            && json.length <= chunk_size && json.length > 0) {

            receiveVM(json, function (e, result) {
                if (e) {
                    callback(e);
                    return;
                }
                VM.log.info('Receive returning: ' + JSON.stringify(result));
                callback(null, result);
            });
        } else {
            VM.log.debug('type: [' + type + ']');
            VM.log.debug('chunk_name: [' + chunk_name + ']');
            VM.log.debug('chunk_size: [' + chunk_size + ']');
            VM.log.debug('json.length: [' + json.length + ']');
            VM.log.warn('Failed to get ' + type + '!');
            callback(new Error('Failed to get ' + type + '!'));
        }
    });
}

exports.receive = function (target, options, callback)
{
    ensureLogging('receive', true);

    VM.log.info('Receiving VM from: ' + JSON.stringify(target));

    if (target.hasOwnProperty('host') && target.hasOwnProperty('port')) {
        // network receive not yet supported either.
        callback(new Error('cannot receive from ' + JSON.stringify(target)));
        return;
    } else if (typeof (target) !== 'string' || target !== '-') {
        callback(new Error('cannot receive from ' + JSON.stringify(target)));
        return;
    }

    receiveStdinChunk('JSON', function (error, result) {
        var eof = false;

        if (error) {
            callback(error);
            return;
        }
        if (result && result === 'EOF') {
            callback(new Error('unable to find JSON in stdin.'));
        } else if (result && result.hasOwnProperty('uuid')) {
            // VM started receive, now need datasets

            VM.log.debug('now looking for datasets');

            async.whilst(
                function () { return !eof; },
                function (cb) {
                    receiveStdinChunk('DATASET', function (err, res) {
                        if (err) {
                            cb(err);
                            return;
                        }
                        if (res === 'EOF') {
                            eof = true;
                        }
                        cb();
                    });
                }, function (err) {
                    if (err) {
                        callback(err);
                        return;
                    }
                    // no error so we read all the datasets, try an install.
                    VM.log.info('receive calling VM.install: ' + eof);
                    VM.install(result.uuid, function (e) {
                        if (e) {
                            VM.log.warn(e, 'couldn\'t install VM: '
                                + e.message);
                        }
                        callback(e, result);
                    });
                }
            );
        } else {
            callback(new Error('unable to receive JSON'));
        }
    });
};

exports.install = function (uuid, callback)
{
    ensureLogging('install', true);

    VM.log.info('Installing VM ' + uuid);

    fs.readFile('/etc/zones/' + uuid + '-receiving.json',
        function (err, data) {
            var payload;

            if (err) {
                callback(err);
                return;
            }

            try {
                payload = JSON.parse(data.toString());
            } catch (e) {
                callback(e);
                return;
            }

            // installZone takes a payload
            installZone(payload, callback);
        }
    );

};

function getAllDatasets(vmobj)
{
    var datasets = [];
    var disk;

    if (vmobj.hasOwnProperty('zfs_filesystem')) {
        datasets.push(vmobj.zfs_filesystem);
    }

    for (disk in vmobj.disks) {
        disk = vmobj.disks[disk];
        if (disk.hasOwnProperty('zfs_filesystem')) {
            datasets.push(disk.zfs_filesystem);
        }
    }

    return datasets;
}

//
// Headers are 512 bytes and look like:
//
// MAGIC-VMBUNDLE\0
// <VERSION>\0 -- ASCII #s
// <CHECKSUM>\0 -- ASCII (not yet used)
// <OBJ-NAME>\0 -- max length: 256
// <OBJ-SIZE>\0 -- ASCII # of bytes
// <PADDED-SIZE>\0 -- ASCII # of bytes, must be multiple of 512
// ...\0
//
function chunkHeader(name, size, padding)
{
    var header = new Buffer(512);
    var pos = 0;

    header.fill(0);
    pos += addString(header, 'MAGIC-VMBUNDLE', pos);
    pos += addString(header, sprintf('%d', 1), pos);
    pos += addString(header, 'CHECKSUM', pos);
    pos += addString(header, name, pos);
    pos += addString(header, sprintf('%d', size), pos);
    pos += addString(header, sprintf('%d', size + padding), pos);

    return (header);
}

// add the string to buffer at pos, returning pos of new end of the buffer.
function addString(buf, str, pos)
{
    var len = str.length;
    buf.write(str, pos);
    return (len + 1);
}

function sendJSON(target, json, cb)
{
    var header;
    var pad;
    var padding = 0;

    if (target === 'stdout') {
        if ((json.length % 512) != 0) {
            padding = 512 - (json.length % 512);
        }
        header = chunkHeader('JSON', json.length, padding);
        process.stdout.write(header);
        process.stdout.write(json, 'ascii');
        if (padding > 0) {
            pad = new Buffer(padding);
            pad.fill(0);
            process.stdout.write(pad);
        }
        cb();
    } else {
        VM.log.error('Don\'t know how to send JSON to '
            + JSON.stringify(target));
        cb(new Error('Don\'t know how to send JSON to '
            + JSON.stringify(target)));
    }
}

function sendDataset(target, dataset, callback)
{
    var header;

    if (target === 'stdout') {

        async.series([
            function (cb) {
                // delete any existing 'sending' snapshot
                zfs(['destroy', '-F', dataset + '@sending'],
                    function (err, fds) {
                        // We don't expect this to succeed, since that means
                        // something left an @sending around. Warn if succeeds.
                        if (!err) {
                            VM.log.warn('Destroyed pre-existing ' + dataset
                                + '@sending');
                        }
                        cb();
                    }
                );
            }, function (cb) {
                zfs(['snapshot', dataset + '@sending'], function (err, fds) {
                    cb(err);
                });
            }, function (cb) {
                header = chunkHeader(dataset, 0, 0);
                process.stdout.write(header);
                cb();
            }, function (cb) {
                var child;

                child = spawn('/usr/sbin/zfs',
                    ['send', '-p', dataset + '@sending'],
                    {customFds: [-1, 1, -1]});
                child.stderr.on('data', function (data) {
                    var idx;
                    var lines = trim(data.toString()).split('\n');

                    for (idx in lines) {
                        VM.log.debug('zfs send: ' + trim(lines[idx]));
                    }
                });
                child.on('close', function (code) {
                    VM.log.debug('zfs send process exited with code '
                        + code);
                    cb();
                });
            }, function (cb) {
                zfs(['destroy', '-F', dataset + '@sending'],
                    function (err, fds) {
                        if (err) {
                            VM.log.warn(err, 'Unable to destroy ' + dataset
                                + '@sending: ' + err.message);
                        }
                        cb(err);
                    }
                );
            }
        ], function (err) {
            if (err) {
                VM.log.error(err, 'Failed to send dataset: ' + err.message);
            } else {
                VM.log.info('Successfully sent dataset');
            }
            callback(err);
        });
    } else {
        VM.log.error('Don\'t know how to send datasets to '
            + JSON.stringify(target));
        callback(new Error('Don\'t know how to send datasets to '
            + JSON.stringify(target)));
    }
}

exports.send = function (uuid, target, options, callback)
{
    var datasets;
    var vmobj;

    ensureLogging('send', true);
    target = 'stdout';

    VM.log.info('Sending VM ' + uuid + ' to: ' + JSON.stringify(target));

    async.series([
        function (cb) {
            // make sure we *can* send first, to avoid wasting cycles
            if (target === 'stdout' && tty.isatty(1)) {
                VM.log.error('Cannot send VM to a TTY.');
                cb(new Error('Cannot send VM to a TTY.'));
            } else {
                cb();
            }
        }, function (cb) {
            VM.load(uuid, function (err, obj) {
                if (err) {
                    cb(err);
                } else {
                    vmobj = obj;
                    cb();
                }
            });
        }, function (cb) {
            datasets = getAllDatasets(vmobj);
            if (datasets.length < 1) {
                VM.log.error('Cannot send VM with no datasets.');
                cb(new Error('VM has no datasets.'));
            } else {
                cb();
            }
        }, function (cb) {
            if (vmobj.state !== 'stopped') {
                // In this case we need to stop it and make sure it stopped.
                VM.stop(uuid, options, function (e) {
                    if (e) {
                        VM.log.error(e, 'unable to stop VM ' + uuid + ': '
                            + e.message);
                        cb(e);
                        return;
                    }

                    VM.load(uuid, function (error, obj) {
                        if (error) {
                            VM.log.error(error, 'unable to reload VM ' + uuid
                                + ': ' + error.message);
                            return;
                        }
                        if (obj.zone_state !== 'installed') {
                            VM.log.error('after stop attempt, state is '
                                + obj.zone_state + ' != installed');
                            cb(new Error('state after stopping is '
                                + obj.zone_state + ' != installed'));
                            return;
                        }
                        cb();
                    });
                });
            } else {
                // already stopped, good to go!
                cb();
            }
        }, function (cb) {
            // Clean up trash left from broken datasets (see OS-388)
            try {
                fs.unlinkSync(vmobj.zonepath + '/SUNWattached.xml');
            } catch (err) {
                // DO NOTHING, this file shouldn't have existed anyway.
            }
            try {
                fs.unlinkSync(vmobj.zonepath + '/SUNWdetached.xml');
            } catch (err) {
                // DO NOTHING, this file shouldn't have existed anyway.
            }
            cb();
        }, function (cb) {
            // send JSON
            var json = JSON.stringify(vmobj, null, 2) + '\n';
            sendJSON(target, json, cb);
        }, function (cb) {
            // send datasets
            async.forEachSeries(datasets, function (ds, c) {
                sendDataset(target, ds, c);
            }, function (e) {
                if (e) {
                    VM.log.error('Failed to send datasets');
                }
                cb(e);
            });
        }
    ], function (err) {
        callback(err);
    });
};

exports.create = function (payload, callback)
{
    ensureLogging('create', true);

    VM.log.info('Creating zone, original payload:\n'
        + JSON.stringify(payload, null, 2));

    normalizePayload(payload, null, function (err) {
        if (err) {
            VM.log.error(err, 'Failed to validate payload: ' + err.message);
            callback(err);
            return;
        }
        VM.log.debug('normalized payload:\n'
            + JSON.stringify(payload, null, 2));

        createZoneUUID(payload, function (e, uuid) {
            if (e) {
                callback(e);
            } else {
                if (BRAND_OPTIONS[payload.brand].features.type === 'KVM') {
                    createVM(payload, function (error, result) {
                        if (error) {
                            callback(error);
                        } else {
                            callback(null, {'uuid': payload.uuid,
                                'zonename': payload.zonename});
                        }
                    });
                } else {
                    createZone(payload, function (error, result) {
                        if (error) {
                            callback(error);
                        } else {
                            callback(null, {'uuid': payload.uuid,
                                'zonename': payload.zonename});
                        }
                    });
                }
            }
        });
    });
};

// delete a zvol
function deleteVolume(volume, callback)
{
    // use recursive delete to handle possible snapshots on volume
    var args = ['destroy', '-rF', volume.zfs_filesystem];

    if (volume.missing) {
        // this volume doesn't actually exist, so skip trying to delete.
        VM.log.info('volume ' + volume.path + ' doesn\'t exist, skipping '
            + 'deletion');
        callback();
        return;
    }

    zfs(args, function (err, fds) {
        // err will be non-null if something broke
        callback(err);
    });
}

function deleteZone(uuid, callback)
{
    var zoneobj;

    async.series([
        function (cb) {
            VM.load(uuid, function (err, obj) {
                if (err) {
                    cb(err);
                }
                zoneobj = obj;
                cb();
            });
        }, function (cb) {
            VM.log.debug('setting autoboot=false');
            zonecfg(['-u', uuid, 'set autoboot=false'], function (e) {
                if (e) {
                    VM.log.warn(e, 'Error setting autoboot=false: '
                        + e.message);
                }
                cb();
            });
        }, function (cb) {
            VM.log.debug('halting zone');
            zoneadm(['-u', uuid, 'halt', '-X'], function (e) {
                if (e) {
                    VM.log.warn(e, 'Error halting zone: ' + e.message);
                }
                cb();
            });
        }, function (cb) {
            VM.log.debug('uninstalling zone');
            zoneadm(['-u', uuid, 'uninstall', '-F'], function (e) {
                if (e) {
                    VM.log.warn(e, 'Error uninstalling zone: ' + e.message);
                }
                cb();
            });
        }, function (cb) {
            if (zoneobj && zoneobj.hasOwnProperty('disks')) {
                async.forEachSeries(zoneobj.disks, deleteVolume,
                    function (err) {
                        if (err) {
                            VM.log.error(err, 'Unknown error deleting volumes: '
                                + err.message);
                            cb(err);
                        } else {
                            VM.log.info('successfully deleted volumes');
                            cb();
                        }
                    }
                );
            } else {
                VM.log.debug('skipping volume destruction for diskless '
                    + zoneobj.uuid);
                cb();
            }
        }, function (cb) {
            if (zoneobj.zonename) {
                VM.log.debug('deleting zone');
                // XXX for some reason -u <uuid> doesn't work with delete
                zonecfg(['-z', zoneobj.zonename, 'delete', '-F'], function (e) {
                    if (e) {
                        VM.log.warn(e, 'Error deleting VM: ' + e.message);
                    }
                    cb();
                });
            } else {
                cb();
            }
        }, function (cb) {
            VM.load(uuid, function (err, obj) {
                var gone = /^zoneadm:.*: No such zone configured/;
                if (err && err.message.match(gone)) {
                    // the zone is gone, that's good.
                    VM.log.debug('confirmed VM is gone.');
                    cb();
                } else if (err) {
                    // there was a non-good error.
                    cb(err);
                } else {
                    // the VM still exists!
                    err = new Error('VM still exists after delete.');
                    err.code = 'EEXIST';
                    cb(err);
                }
            });
        }, function (cb) {
            // delete the incoming payload if it exists
            fs.unlink('/etc/zones/' + zoneobj.uuid + '-receiving.json',
                function (e) {
                    // we can't do anyhing if this fails other than log
                    if (e && e.code !== 'ENOENT') {
                        VM.log.warn(e, 'Failed to delete ' + zoneobj.uuid
                            + '-receiving.json (' + e.code + '): ' + e.message);
                    }
                    cb();
                }
            );
        }
    ], function (error) {
        callback(error);
    });
}

exports.delete = function (uuid, callback)
{
    var attemptDelete;
    var last_try = 16;
    var next_try = 1;
    var tries = 0;

    ensureLogging('delete', true);
    VM.log.info('Deleting VM ' + uuid);

    attemptDelete = function (cb) {
        next_try = (next_try * 2);
        deleteZone(uuid, function (err) {
            tries++;
            if (err && err.code === 'EEXIST') {
                // zone still existed, try again if we've not tried too much.
                if (next_try <= last_try) {
                    VM.log.info('VM.delete(' + tries + '): still there, '
                        + 'will try again in: ' + next_try + ' secs');
                    setTimeout(function () {
                        // try again
                        attemptDelete(cb);
                    }, next_try * 1000);
                } else {
                    VM.log.warn('VM.delete(' + tries + '): still there after'
                        + ' ' + next_try + ' seconds, giving up.');
                    cb(new Error('delete failed after ' + tries + ' attempts. '
                        + '(check the log for details)'));
                    return;
                }
            } else if (err) {
                // error but not one we can retry from.
                VM.log.error(err, 'VM.delete: FATAL: ' + err.message);
                cb(err);
            } else {
                // success!
                VM.log.debug('VM.delete: SUCCESS');
                cb();
            }
        });
    };

    attemptDelete(function (err) {
        if (err) {
            VM.log.error(err);
        }
        callback(err);
    });
};

function startZone(zoneobj, callback)
{
    var set_autoboot = 'set autoboot=true';
    var uuid = zoneobj.uuid;

    VM.log.debug('startZone starting ' + uuid);

    //
    // We set autoboot (or vm-autoboot) here because we've just intentionally
    // started this vm, so we want it to come up if the host is rebooted.
    //
    if (BRAND_OPTIONS[zoneobj.brand].features.use_vm_autoboot) {
        set_autoboot = 'select attr name=vm-autoboot; set value=true; end';
    }

    zoneadm(['-u', uuid, 'boot', '-X'], function (err, boot_fds) {
        if (err) {
            VM.log.error(err);
            callback(err);
            return;
        }
        zonecfg(['-u', uuid, set_autoboot], function (e, autoboot_fds) {
            if (e) {
                // The vm is running at this point, erroring out here would
                // do no good, so we just log it.
                VM.log.error(e, 'startZone(): Failed to ' + set_autoboot
                    + ' for ' + uuid + ': ' + e.message);
            }

            if (zoneobj.never_booted) {
                zonecfg(['-u', uuid, 'remove attr name=never-booted' ],
                    function (error, neverbooted_fds) {
                        // Ignore errors here, because we're started.
                        if (error) {
                            VM.log.warn(error, 'failed to remove '
                                + 'never-booted flag for ' + uuid + ': '
                                + error.message);
                        }
                        callback();
                    }
                );
            } else {
                VM.log.info('Started ' + uuid);
                callback();
            }
        });
    });
}

// build the qemu cmdline and start up a VM
function startVM(vmobj, extra, callback)
{
    var check_path;
    var cmdargs = [];
    var d;
    var defaultgw = '';
    var disk;
    var diskargs = '';
    var disk_idx = 0;
    var found;
    var hostname = vmobj.uuid;
    var mdata;
    var nic;
    var nic_idx = 0;
    var primary_found = false;
    var qemu_opts = '';
    var r;
    var script;
    var spiceargs;
    var uuid = vmobj.uuid;
    var virtio_txburst;
    var virtio_txtimer;
    var vnic_opts;

    VM.log.debug('startVM(' + uuid + ')');

    if (!vmobj.hasOwnProperty('state')) {
        callback(new Error('Cannot start VM ' + uuid + ' which has no state'));
        return;
    }

    if ((vmobj.state !== 'stopped' && vmobj.state !== 'provisioning')
        || (vmobj.state === 'provisioning'
        && vmobj.zone_state !== 'installed')) {

        callback(new Error('Cannot start VM from state: ' + vmobj.state
            + ', must be "stopped"'));
        return;
    }

    // XXX TODO: validate vmobj data is ok to start

    cmdargs.push('-m', vmobj.ram);
    cmdargs.push('-name', vmobj.uuid);
    cmdargs.push('-uuid', vmobj.uuid);

    if (vmobj.hasOwnProperty('cpu_type')) {
        cmdargs.push('-cpu', vmobj.cpu_type);
    } else {
        cmdargs.push('-cpu', 'qemu64');
    }

    if (vmobj.vcpus > 1) {
        cmdargs.push('-smp', vmobj.vcpus);
    }

    for (disk in vmobj.disks) {
        if (vmobj.disks.hasOwnProperty(disk)) {
            disk = vmobj.disks[disk];
            if (!disk.media) {
                disk.media = 'disk';
            }
            diskargs = 'file=' + disk.path + ',if=' + disk.model
                + ',index=' + disk_idx + ',media=' + disk.media;
            if (disk.boot) {
                diskargs = diskargs + ',boot=on';
            }
            cmdargs.push('-drive', diskargs);
            disk_idx++;
        }
    }

    // extra payload can include additional disks that we want to include only
    // on this one boot.  It can also contain a boot parameter to control boot
    // device.  See qemu http://qemu.weilnetz.de/qemu-doc.html for info on
    // -boot options.
    if (extra.hasOwnProperty('disks')) {
        for (disk in extra.disks) {
            if (extra.disks.hasOwnProperty(disk)) {
                disk = extra.disks[disk];

                // ensure this is either a disk that gets mounted in or a
                // file that's been dropped in to the zonepath
                found = false;
                for (d in vmobj.disks) {
                    if (!found && vmobj.disks.hasOwnProperty(d)) {
                        d = vmobj.disks[d];
                        if (d.path === disk.path) {
                            found = true;
                        }
                    }
                }
                check_path = path.join(vmobj.zonepath, 'root', disk.path);
                if (!found && fs.existsSync(check_path)) {
                    found = true;
                }
                if (!found) {
                    callback(new Error('Cannot find disk: ' + disk.path));
                    return;
                }

                if (!disk.media) {
                    disk.media = 'disk';
                }
                diskargs = 'file=' + disk.path + ',if=' + disk.model
                    + ',index=' + disk_idx + ',media=' + disk.media;
                if (disk.boot) {
                    diskargs = diskargs + ',boot=on';
                }
                cmdargs.push('-drive', diskargs);
                disk_idx++;
            }
        }
    }

    // helpful values:
    // order=nc (network boot, then fallback to disk)
    // once=d (boot on disk once and the fallback to default)
    // order=c,once=d (boot on CDROM this time, but not subsequent boots)
    if (extra.hasOwnProperty('boot')) {
        cmdargs.push('-boot', extra.boot);
    } else if (vmobj.hasOwnProperty('boot')) {
        cmdargs.push('-boot', vmobj.boot);
    } else {
        // order=cd means try harddisk first (c) and cdrom if that fails (d)
        cmdargs.push('-boot', 'order=cd');
    }

    if (vmobj.hasOwnProperty('hostname')) {
        hostname = vmobj.hostname;
    }

    if (vmobj.hasOwnProperty('default_gateway')) {
        defaultgw = vmobj['default_gateway'];
    }

    /*
     * These tunables are set for all virtio vnics on this VM.
     */
    virtio_txtimer = VIRTIO_TXTIMER_DEFAULT;
    virtio_txburst = VIRTIO_TXBURST_DEFAULT;
    if (vmobj.hasOwnProperty('virtio_txtimer')) {
        virtio_txtimer = vmobj.virtio_txtimer;
    }
    if (vmobj.hasOwnProperty('virtio_txburst')) {
        virtio_txburst = vmobj.virtio_txburst;
    }

    for (nic in vmobj.nics) {
        if (vmobj.nics.hasOwnProperty(nic)) {
            nic = vmobj.nics[nic];

            // for virtio devices, we want to be able to set the txtimer and
            // txburst so we use a '-device' instead of a '-net' line.
            if (nic.model === 'virtio') {
                cmdargs.push('-device',
                    'virtio-net-pci,mac=' + nic.mac
                    + ',tx=timer,x-txtimer=' + virtio_txtimer
                    + ',x-txburst=' + virtio_txburst
                    + ',vlan=' + nic_idx);
            } else {
                cmdargs.push('-net',
                    'nic,macaddr=' + nic.mac
                    + ',vlan=' + nic_idx
                    + ',name=net' + nic_idx
                    + ',model=' + nic.model);
            }
            vnic_opts = 'vnic,name=net' + nic_idx
                + ',vlan=' + nic_idx
                + ',ifname=net' + nic_idx;

            if (nic.ip != 'dhcp') {
                vnic_opts = vnic_opts
                    + ',ip=' + nic.ip
                    + ',netmask=' + nic.netmask;
            }

            // The primary network provides the resolvers, default gateway
            // and hostname to prevent vm from trying to use settings
            // from more than one nic
            if (!primary_found) {
                if (nic.hasOwnProperty('primary') && nic.primary) {
                    if (nic.hasOwnProperty('gateway') && nic.ip != 'dhcp') {
                        vnic_opts += ',gateway_ip=' + nic.gateway;
                    }
                    primary_found = true;
                } else if (defaultgw && nic.hasOwnProperty('gateway')
                    && nic.gateway == defaultgw) {

                    /*
                     * XXX this exists here for backward compatibilty.  New VMs
                     *     and old VMs that are upgraded should not use
                     *     default_gateway.  When we've implemented autoupgrade
                     *     this block (and all reference to default_gateway)
                     *     can be removed.
                     */

                    if (nic.ip != 'dhcp') {
                        vnic_opts += ',gateway_ip=' + nic.gateway;
                    }
                    primary_found = true;
                }

                if (primary_found && nic.ip != 'dhcp') {
                    if (hostname) {
                        vnic_opts += ',hostname=' + hostname;
                    }
                    if (vmobj.hasOwnProperty('resolvers')) {
                        for (r in vmobj.resolvers) {
                            vnic_opts += ',dns_ip' + r + '='
                                + vmobj.resolvers[r];
                        }
                    }
                }
            }

            cmdargs.push('-net', vnic_opts);
            nic_idx++;
        }
    }

    cmdargs.push('-smbios', 'type=1,manufacturer=Joyent,'
        + 'product=SmartDC HVM,version=6.2012Q1,'
        + 'serial=' + vmobj.uuid + ',uuid=' + vmobj.uuid + ','
        + 'sku=001,family=Virtual Machine');

    cmdargs.push('-pidfile', '/tmp/vm.pid');

    if (vmobj.hasOwnProperty('vga')) {
        cmdargs.push('-vga', vmobj.vga);
    } else {
        cmdargs.push('-vga', 'cirrus');
    }

    cmdargs.push('-chardev',
        'socket,id=qmp,path=/tmp/vm.qmp,server,nowait');
    cmdargs.push('-qmp', 'chardev:qmp');

    // serial0 is for serial console
    cmdargs.push('-chardev',
        'socket,id=serial0,path=/tmp/vm.console,server,nowait');
    cmdargs.push('-serial', 'chardev:serial0');

    // serial1 is used for metadata API
    cmdargs.push('-chardev',
        'socket,id=serial1,path=/tmp/vm.ttyb,server,nowait');
    cmdargs.push('-serial', 'chardev:serial1');

    if (!vmobj.qemu_opts) {
        if (vmobj.hasOwnProperty('vnc_password')
            && vmobj.vnc_password.length > 0) {

            cmdargs.push('-vnc', 'unix:/tmp/vm.vnc,password');
        } else {
            cmdargs.push('-vnc', 'unix:/tmp/vm.vnc');
        }
        if (vmobj.hasOwnProperty('spice_port')
            && vmobj.spice_port !== -1) {

            spiceargs = 'sock=/tmp/vm.spice';
            if (!vmobj.hasOwnProperty('spice_password')
                || vmobj.spice_password.length <= 0) {

                spiceargs = spiceargs + ',disable-ticketing';

                // Otherwise, spice password is set via qmp, so we don't
                // need to do anything here
            }
            if (vmobj.hasOwnProperty('spice_opts')
                && vmobj.spice_opts.length > 0) {

                spiceargs = spiceargs + ',' + vmobj.spice_opts;
            }
            cmdargs.push('-spice', spiceargs);
        }
        cmdargs.push('-parallel', 'none');
        cmdargs.push('-usb');
        cmdargs.push('-usbdevice', 'tablet');
        cmdargs.push('-k', 'en-us');
    } else {
        qemu_opts = vmobj.qemu_opts.toString();
    }

    if (vmobj.qemu_extra_opts) {
        qemu_opts = qemu_opts + ' ' + vmobj.qemu_extra_opts;
    }

    // This actually creates the qemu process
    script = '#!/usr/bin/bash\n\n'
        + 'exec >/tmp/vm.startvm.log 2>&1\n\n'
        + 'set -o xtrace\n\n'
        + 'if [[ -x /startvm.zone ]]; then\n'
        + '    exec /smartdc/bin/qemu-exec /startvm.zone "'
        + cmdargs.join('" "')
        + '" ' + qemu_opts + '\n'
        + 'else\n'
        + '    exec /smartdc/bin/qemu-exec /smartdc/bin/qemu-system-x86_64 "'
        + cmdargs.join('" "')
        + '" ' + qemu_opts + '\n'
        + 'fi\n\n'
        + 'exit 1\n';

    try {
        fs.writeFileSync(vmobj.zonepath + '/root/startvm', script);
        fs.chmodSync(vmobj.zonepath + '/root/startvm', '0755');
    } catch (e) {
        VM.log.warn(e, 'Unable to create /startvm script in ' + vmobj.uuid);
        callback(new Error('cannot create /startvm'));
        return;
    }

    mdata = {
        'internal_metadata':
            vmobj.internal_metadata ? vmobj.internal_metadata : {}
    };
    fs.writeFile(path.join(vmobj.zonepath, '/root/tmp/vm.metadata'),
        JSON.stringify(mdata, null, 2) + '\n',
        function (err) {
            if (err) {
                VM.log.debug(err, 'FAILED TO write metadata to '
                    + '/tmp/vm.metadata: ' + err.message);
                callback(err);
            } else {
                VM.log.debug('wrote metadata to /tmp/vm.metadata');
                startZone(vmobj, callback);
            }
        }
    );
}

exports.start = function (uuid, extra, callback)
{
    var zoneobj;

    ensureLogging('start', true);
    VM.log.info('Starting VM ' + uuid);

    VM.load(uuid, function (err, obj) {
        if (err) {
            callback(err);
        } else {

            if ((obj.state !== 'stopped' && obj.state !== 'provisioning')
                || (obj.state === 'provisioning'
                && obj.zone_state !== 'installed')) {

                err = new Error('Cannot to start vm from state "' + obj.state
                    + '", must be "stopped".');
                VM.log.error(err);
                callback(err);
                return;
            }

            zoneobj = obj;

            lookupInvalidNicTags(obj.nics, function (e) {
                if (e) {
                    callback(e);
                    return;
                }

                if (BRAND_OPTIONS[obj.brand].features.type === 'KVM') {
                    startVM(zoneobj, extra, callback);
                } else if (BRAND_OPTIONS[obj.brand].features.type === 'OS') {
                    startZone(zoneobj, callback);
                } else {
                    err = new Error('no idea how to start a vm with brand: '
                        + obj.brand);
                    VM.log.error(err);
                    callback(err);
                }
            });
        }
    });
};

function setRctl(zonename, rctl, value, callback)
{
    var args;

    args = ['-n', rctl, '-v', value.toString(), '-r', '-i', 'zone', zonename];
    VM.log.debug('/usr/bin/prctl ' + args.join(' '));
    execFile('/usr/bin/prctl', args, function (error, stdout, stderr) {
        if (error) {
            VM.log.error(error, 'setRctl() failed with: ' + stderr);
            callback(error);
        } else {
            callback();
        }
    });
}

function resizeTmp(zonename, newsize, callback)
{
    var args;
    var newvfsline = 'swap        -   /tmp                tmpfs    -   yes    '
        + 'size=' + newsize + 'm';
    var vfsreplace = '"s|^swap.*/tmp.*tmpfs.*size=.*$|' + newvfsline + '|"';

    args = [zonename, '/usr/sbin/mount', '-F', 'tmpfs', '-o', 'remount,size='
        + newsize + 'm', '/tmp'];
    VM.log.debug('/usr/sbin/zlogin ' + args.join(' '));
    execFile('/usr/sbin/zlogin', args, function (err, mnt_stdout, mnt_stderr) {
        if (err) {
            VM.log.error({'err': err, 'stdout': mnt_stdout,
                'stderr': mnt_stderr}, 'zlogin for ' + zonename
                + ' exited with code' + err.code + err.message);
            // error here is not fatal as we want to still try to fix for next
            // reboot.
        }

        args = [zonename, '/usr/bin/sed', '-i', '""', '-e', vfsreplace,
            '/etc/vfstab'];
        VM.log.debug('/usr/sbin/zlogin ' + args.join(' '));
        execFile('/usr/sbin/zlogin', args,
            function (e, sed_stdout, sed_stderr) {

                if (e) {
                    VM.log.error({'err': e, 'stdout': sed_stdout,
                        'stderr': sed_stderr}, 'zlogin for ' + zonename + ' '
                        + 'exited with ' + 'code ' + e.code + ': ' + e.message);
                    // error here is not fatal as there's nothing really to do
                    // about it.
            }
            callback();
        });
    });
}

function resizeDisks(disks, updates, callback)
{
    var d;
    var disk;
    var resized = 0;
    var vols = [];

    for (disk in updates) {
        disk = updates[disk];
        for (d in disks) {
            d = disks[d];
            if (d.path === disk.path && disk.hasOwnProperty('size')) {
                vols.push({'disk': d, 'new_size': disk.size});
            }
        }
    }

    function resize(vol, cb) {
        var args;
        var dsk = vol.disk;
        var size = vol.new_size;

        if (dsk.hasOwnProperty('zfs_filesystem')) {
            if (dsk.size > size) {
                cb(new Error('cannot resize ' + dsk.zfs_filesystem
                    + ' new size must be greater than current size. ('
                    + dsk.size + ' > ' + dsk.size + ')'));
            } else if (dsk.size === size) {
                // no point resizing if the old+new are the same
                cb();
            } else {
                args = ['set', 'volsize=' + size + 'M', dsk.zfs_filesystem];
                zfs(args, function (err, fds) {
                    resized++;
                    cb(err);
                });
            }
        } else {
            cb(new Error('could not find zfs_filesystem in '
                + JSON.stringify(dsk)));
        }
    }

    async.forEachSeries(vols, resize, function (err) {
        if (err) {
            VM.log.error(err, 'Unable to resize disks');
            callback(err);
        } else {
            callback(null, resized);
        }
    });
}

function updateVnicAllowedIPs(uuid, nic, callback)
{
    var ips = [];
    if (!uuid || !nic.interface) {
        callback();
        return;
    }

    if (nic.hasOwnProperty('ip')) {
        ips.push(nic.ip);
    }

    if (nic.hasOwnProperty('vrrp_primary_ip')) {
        ips.push(nic.vrrp_primary_ip);
    }

    if (!ips.length === 0) {
        dladm.resetLinkProp(uuid, nic.interface, 'allowed-ips', VM.log,
            callback);
    } else {
        dladm.setLinkProp(uuid, nic.interface, 'allowed-ips', ips, VM.log,
            callback);
    }
}

function updateVnicProperties(uuid, vmobj, payload, callback)
{
    if (vmobj.state != 'running') {
        VM.log.debug('VM not running: not updating vnic properties');
        callback(null);
        return;
    }

    if (!payload.hasOwnProperty('update_nics')) {
        VM.log.debug(
            'No update_nics property: not updating vnic properties');
        callback(null);
        return;
    }

    async.forEach(payload.update_nics, function (nic, cb) {
        var opt;
        var needsUpdate = false;
        var needsIPupdate = false;
        var spoof_opts = {
            'allow_ip_spoofing': 'ip-nospoof',
            'allow_mac_spoofing': 'mac-nospoof',
            'allow_dhcp_spoofing': 'dhcp-nospoof',
            'allow_restricted_traffic': 'restricted'
        };
        var vm_nic;

        // First, determine if we've changed any of the spoofing opts in this
        // update:
        for (opt in spoof_opts) {
            if (nic.hasOwnProperty(opt)) {
                needsUpdate = true;
                break;
            }
        }

        if (nic.hasOwnProperty('vrrp_primary_ip')) {
            needsIPupdate = true;
        }

        for (vm_nic in vmobj.nics) {
            vm_nic = vmobj.nics[vm_nic];
            if (vm_nic.mac == nic.mac) {
                break;
            }
        }
        if (!vm_nic) {
            cb(new Error('Unknown NIC: ' + nic.mac));
            return;
        }

        if (!needsUpdate) {
            VM.log.debug('No spoofing / allowed IP opts updated for nic "'
                + nic.mac + '": not updating');
            if (needsIPupdate) {
                updateVnicAllowedIPs(uuid, vm_nic, cb);
            } else {
                cb(null);
            }
            return;
        }

        // Using the updated nic object, figure out what spoofing opts to set
        for (opt in spoof_opts) {
            if (vm_nic.hasOwnProperty(opt) && fixBoolean(vm_nic[opt])) {
                delete spoof_opts[opt];
            }
        }

        if (vm_nic.hasOwnProperty('dhcp_server')
                && fixBoolean(vm_nic.dhcp_server)) {
            delete spoof_opts.allow_dhcp_spoofing;
            delete spoof_opts.allow_ip_spoofing;
        }

        if (Object.keys(spoof_opts).length === 0) {
            dladm.resetLinkProp(uuid, vm_nic.interface, 'protection', VM.log,
                function (err) {
                    if (err) {
                        cb(err);
                        return;
                    }
                    if (needsIPupdate) {
                        updateVnicAllowedIPs(uuid, vm_nic, cb);
                        return;
                    }
                    cb();
                    return;
                });
        } else {
            dladm.setLinkProp(uuid, vm_nic.interface, 'protection',
                    Object.keys(spoof_opts).map(function (k) {
                        return spoof_opts[k];
                    }), VM.log,
                function (err) {
                    if (err) {
                        cb(err);
                        return;
                    }
                    if (needsIPupdate) {
                        updateVnicAllowedIPs(uuid, vm_nic.interface,
                            nic.vrrp_primary_ip, cb);
                        return;
                    }
                    cb();
                    return;
                });
        }
    }, function (err) {
        if (err) {
            callback(err);
        } else {
            callback(null);
        }
    });
}

function applyUpdates(oldobj, newobj, payload, callback)
{
    var changed_datasets = false;

    // Note: oldobj is the VM *before* the update, newobj *after*

    async.series([
        function (cb) {
            if (payload.hasOwnProperty('update_disks')
                && oldobj.hasOwnProperty('disks')) {

                resizeDisks(oldobj.disks, payload.update_disks,
                    function (err, resized) {
                        // If any were resized, mark that we changed something
                        if (!err && resized > 0) {
                            changed_datasets = true;
                        }
                        cb(err);
                    }
                );
            } else {
                cb();
            }
        }, function (cb) {
            if (payload.hasOwnProperty('quota')
                && (Number(payload.quota) !== Number(oldobj.quota))) {

                setQuota(newobj.zfs_filesystem, payload.quota, function (err) {
                    if (!err) {
                        changed_datasets = true;
                    }
                    cb(err);
                });
            } else {
                cb();
            }
        }, function (cb) {
            // NOTE: we've already validated the value
            if (payload.hasOwnProperty('zfs_root_recsize')
                && (payload.zfs_root_recsize !== oldobj.zfs_root_recsize)) {

                zfs(['set', 'recsize=' + payload.zfs_root_recsize,
                    newobj.zfs_filesystem], function (err, fds) {

                    if (err) {
                        VM.log.error(err, 'failed to apply zfs_root_recsize: '
                            + fds.stderr);
                        cb(new Error(rtrim(fds.stderr)));
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            // NOTE: we've already validated the value.
            if (payload.hasOwnProperty('zfs_data_recsize')
                && oldobj.hasOwnProperty('zfs_data_recsize')
                && newobj.hasOwnProperty('datasets')
                && (newobj.datasets.indexOf(newobj.zfs_filesystem
                    + '/data') !== -1)) {

                zfs(['set', 'recsize=' + payload.zfs_data_recsize,
                    newobj.zfs_filesystem + '/data'], function (err, fds) {

                    if (err) {
                        VM.log.error(err, 'failed to apply zfs_data_recsize: '
                            + fds.stderr);
                        cb(new Error(rtrim(fds.stderr)));
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            // NOTE: we've already validated the value
            if (payload.hasOwnProperty('zfs_root_compression')
                && (payload.zfs_root_compression !==
                    oldobj.zfs_root_compression)) {

                zfs(['set', 'compression=' + payload.zfs_root_compression,
                    newobj.zfs_filesystem], function (err, fds) {

                    if (err) {
                        VM.log.error(err, 'failed to apply '
                            + 'zfs_root_compression: ' + fds.stderr);
                        cb(new Error(rtrim(fds.stderr)));
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            // NOTE: we've already validated the value
            if (payload.hasOwnProperty('zfs_data_compression')
                && newobj.hasOwnProperty('datasets')
                && (newobj.datasets.indexOf(newobj.zfs_filesystem
                    + '/data') !== -1)) {

                zfs(['set', 'compression=' + payload.zfs_data_compression,
                    newobj.zfs_filesystem + '/data'], function (err, fds) {

                    if (err) {
                        VM.log.error(err, 'failed to apply '
                            + 'zfs_data_compression: ' + fds.stderr);
                        cb(new Error(rtrim(fds.stderr)));
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            var d;
            if (payload.hasOwnProperty('update_disks')) {
                // loop through the disks we updated and perform any updates.
                async.forEachSeries(payload.update_disks, function (disk, c) {
                    for (d in oldobj.disks) {
                        d = oldobj.disks[d];
                        if (d.path === disk.path
                            && disk.hasOwnProperty('compression')
                            && oldobj.hasOwnProperty('zfs_filesystem')) {

                            zfs(['set', 'compression=' + disk.compression,
                                oldobj.zfs_filesystem], function (err, fds) {

                                c(err);
                            });
                        }
                    }
                }, cb);
            } else {
                cb();
            }
        }, function (cb) {
            var filename;
            var n;

            if (!BRAND_OPTIONS[newobj.brand].features.write_zone_netfiles) {
                // this brand doesn't want us writing its net files
                cb();
                return;
            }

            for (n in newobj.nics) {
                n = newobj.nics[n];

                filename = newobj.zonepath + '/root/etc/hostname.'
                    + n.interface;
                if (n.ip !== 'dhcp') {
                    fs.writeFileSync(filename, n.ip + ' netmask ' + n.netmask
                        + ' up' + '\n');
                } else {
                    // try to unlink if it exists, error is non-fatal.
                    try {
                        fs.unlinkSync(filename);
                    } catch (e) {
                        // don't care
                    }
                }

                filename = newobj.zonepath + '/root/etc/dhcp.'
                    + n.interface;
                // try to unlink if it exists, error is non-fatal.
                try {
                    fs.unlinkSync(filename);
                } catch (e) {
                    // don't care
                }

                if (n.hasOwnProperty('primary')) {
                    if (n.hasOwnProperty('gateway')) {
                        fs.writeFileSync(newobj.zonepath + '/root/etc/'
                            + 'defaultrouter', n.gateway + '\n');
                    }
                    if (n.ip === 'dhcp') {
                        fs.writeFileSync(filename, '');
                    }
                }
            }

            // XXX we leave old hostname.netX files around and just replace when
            //     we have one next.

            cb();
        }, function (cb) {
            var factor;
            var keys = [];
            var rctl;
            var rctls = {
                'cpu_shares': ['zone.cpu-shares'],
                'zfs_io_priority': ['zone.zfs-io-priority'],
                'max_lwps': ['zone.max-lwps'],
                'max_physical_memory': ['zone.max-physical-memory',
                    (1024 * 1024)],
                'max_locked_memory': ['zone.max-locked-memory', (1024 * 1024)],
                'max_swap': ['zone.max-swap', (1024 * 1024)],
                'cpu_cap': ['zone.cpu-cap']
            };

            if (!BRAND_OPTIONS[oldobj.brand].features.update_rctls) {
                cb();
                return;
            }

            for (rctl in rctls) {
                keys.push(rctl);
            }

            async.forEachSeries(keys, function (prop, c) {
                rctl = rctls[prop][0];
                if (rctls[prop][1]) {
                    factor = rctls[prop][1];
                } else {
                    factor = 1;
                }

                if (payload.hasOwnProperty(prop)) {
                    setRctl(newobj.zonename, rctl,
                        Number(payload[prop]) * factor,
                        function (err) {
                            if (err) {
                                VM.log.warn(err, 'failed to set rctl: ' + prop);
                            }
                            c();
                        }
                    );
                } else {
                    c();
                }
            }, function (err) {
                cb(err);
            });
        }, function (cb) {
            if ((payload.hasOwnProperty('vnc_password')
                && (oldobj.vnc_password !== newobj.vnc_password))
                || (payload.hasOwnProperty('vnc_port')
                    && (oldobj.vnc_port !== newobj.vnc_port))) {

                // tell vmadmd to refresh_password and port (will restart
                // listener)
                postVmadmd(newobj.uuid, 'reload_display', {}, function (e) {
                    if (e) {
                        cb(new Error('Unable to tell vmadmd to reload VNC'));
                    } else {
                        cb();
                    }
                });
            } else if ((payload.hasOwnProperty('spice_password')
                && (oldobj.spice_password !== newobj.spice_password))
                || (payload.hasOwnProperty('spice_port')
                    && (oldobj.spice_port !== newobj.spice_port))) {

                // tell vmadmd to refresh_password and port (will restart
                // listener)
                postVmadmd(newobj.uuid, 'reload_display', {}, function (e) {
                    if (e) {
                        cb(new Error('Unable to tell vmadmd to reload SPICE'));
                    } else {
                        cb();
                    }
                });
            } else {
                cb();
            }
        }, function (cb) {
            // we do this last, since we need the memory in the zone updated
            // first if we're growing this.
            if (payload.hasOwnProperty('tmpfs')) {
                resizeTmp(newobj.zonename, payload.tmpfs, cb);
            } else {
                cb();
            }
        }, function (cb) {
            var now = new Date();

            // If we changed any dataset properties, we touch the zone's xml
            // file so that last_modified is correct.
            if (changed_datasets && newobj.hasOwnProperty('zonename')) {
                fs.utimes('/etc/zones/' + newobj.zonename + '.xml', now, now,
                    function (err) {
                        if (err) {
                            VM.log.warn(err, 'Unable to "touch" xml file for "'
                                + newobj.zonename + '": ' + err.message);
                        } else {
                            VM.log.debug('Touched ' + newobj.zonename
                                + '.xml after datasets were modified.');
                        }
                        // We don't error out if we just couldn't touch because
                        // the actual updates above already did happen.
                        cb();
                    }
                );
            } else {
                cb();
            }
        }

    ], function (err, res) {
        callback(err);
    });
}

exports.update = function (uuid, payload, callback)
{
    var new_vmobj;
    var vmobj;

    ensureLogging('update', true);

    VM.log.info('Updating VM ' + uuid + ' with initial payload:\n'
        + JSON.stringify(payload, null, 2));

    async.series([
        function (cb) {
            VM.load(uuid, function (err, obj) {
                if (err) {
                    cb(err);
                    return;
                }
                vmobj = obj;
                cb();
            });
        }, function (cb) {
            normalizePayload(payload, vmobj, function (e) {
                VM.log.debug('Used payload:\n'
                    + JSON.stringify(payload, null, 2));
                cb(e);
            });
        }, function (cb) {
            var deletables = [];
            var to_remove = [];
            var n;

            // destroy remove_disks before we add in case we're recreating with
            // an existing name.

            if (payload.hasOwnProperty('remove_disks')) {
                to_remove = payload.remove_disks;
                for (n in vmobj.disks) {
                    if (to_remove.indexOf(vmobj.disks[n].path) !== -1) {
                        deletables.push(vmobj.disks[n]);
                    }
                }
            } else {
                // no disks to remove so all done.
                cb();
                return;
            }

            async.forEachSeries(deletables, deleteVolume,
                function (err) {
                    if (err) {
                        VM.log.error(err, 'Unknown error deleting volumes: '
                            + err.message);
                        cb(err);
                    } else {
                        VM.log.info('successfully deleted volumes');
                        cb();
                    }
                }
            );
        }, function (cb) {
            var disks = [];
            var matches;
            var n;
            var p;
            var used_disk_indexes = [];

            // create any new volumes we need.
            if (payload.hasOwnProperty('add_disks')) {
                disks = payload.add_disks;
            }

            // create a list of used indexes so we can find the free ones to
            // use in createVolume()
            if (vmobj.hasOwnProperty('disks')) {
                for (n in vmobj.disks) {
                    matches = vmobj.disks[n].path.match(/^.*-disk(\d+)$/);
                    if (matches) {
                        used_disk_indexes.push(Number(matches[1]));
                    }
                }
            }

            // add the bits of payload createVolumes() needs.
            p = {'add_disks': disks};
            p.uuid = uuid;
            if (vmobj.hasOwnProperty('zpool')) {
                p.zpool = vmobj.zpool;
            }
            p.used_disk_indexes = used_disk_indexes;
            createVolumes(p, function (e) {
                cb(e);
            });
        }, function (cb) {
            updateMetadata(vmobj, payload, function (e) {
                cb(e);
            });
        }, function (cb) {
            var zcfg;
            // generate a payload and send as a file to zonecfg to update
            // the zone.
            zcfg = buildZonecfgUpdate(vmobj, payload);
            zonecfgFile(zcfg, ['-u', uuid], function (e, fds) {
                cb(e);
            });
        }, function (cb) {
            updateVnicProperties(uuid, vmobj, payload, function (e) {
                cb(e);
            });
        }, function (cb) {
            VM.load(uuid, function (e, newobj) {
                if (e) {
                    cb(e);
                } else {
                    new_vmobj = newobj;
                    cb();
                }
            });
        }, function (cb) {
            applyUpdates(vmobj, new_vmobj, payload, function () {
                cb();
            });
        }
    ], function (e) {
        callback(e);
    });
};

function kill(uuid, callback)
{
    var unset_autoboot = 'set autoboot=false';

    VM.log.info('Killing VM ' + uuid);

    /* We load here to ensure this vm exists. */
    VM.load(uuid, function (err, obj) {
        if (err) {
            callback(err);
            return;
        }

        if (BRAND_OPTIONS[obj.brand].features.use_vm_autoboot) {
            unset_autoboot =
                'select attr name=vm-autoboot; set value=false; end';
        }

        zoneadm(['-u', uuid, 'halt', '-X'], function (e, fds) {
            var msg = trim(fds.stderr);

            if (msg.match(/zone is already halted$/)) {
                // remove transition marker, since vm is not running now.
                unsetTransition(obj, function () {
                    callback(null, 'vm is not running');
                });
            } else if (e) {
                VM.log.error(e, 'failed to kill ' + uuid + ': ' + fds.stderr);
                callback(err, msg);
            } else {
                zonecfg(['-u', uuid, unset_autoboot],
                    function (error, unset_fds) {

                    if (error) {
                        // The vm is dead at this point, erroring out here would
                        // do no good, so we just log it.
                        VM.log.error('killVM(): Failed to '
                            + unset_autoboot + ' for ' + uuid);
                    }
                    if (obj.state === 'stopping') {
                        // remove transition marker
                        unsetTransition(obj, function () {
                            callback(null, msg);
                        });
                    } else {
                        callback(null, msg);
                    }
                });
            }
        });
    });
}

function postVmadmd(uuid, action, args, callback)
{
    var arg;
    var url_path = '/vm/' + uuid + '?action=' + action;
    var req;

    if (args) {
        for (arg in args) {
            if (args.hasOwnProperty(arg)) {
                url_path = url_path + '&' + arg + '=' + args[arg];
            }
        }
    }

    VM.log.debug('HTTP POST ' + url_path);
    req = http.request(
        { method: 'POST', host: '127.0.0.1', port: '8080', path: url_path },
        function (res) {

            VM.log.debug('HTTP STATUS: ' + res.statusCode);
            VM.log.debug('HTTP HEADERS: ' + JSON.stringify(res.headers));
            res.setEncoding('utf8');
            res.on('data', function (chunk) {
                VM.log.debug('HTTP BODY: ' + chunk);
            });
            res.on('end', function () {
                VM.log.debug('HTTP conversation has completed.');
                callback();
            });
        }
    );
    req.on('error', function (e) {
        VM.log.error(e, 'HTTP error: ' + e.message);
        callback(e);
    });
    req.end();
}

exports.stop = function (uuid, options, callback)
{
    var unset_autoboot = 'set autoboot=false';
    var vmobj;


    if (options.hasOwnProperty('force') && options.force) {
        ensureLogging('stop-F', true);
        kill(uuid, callback);
        return;
    } else {
        ensureLogging('stop', true);
    }

    VM.log.info('Stopping VM ' + uuid);

    if (!options) {
        options = {};
    }
    if (!options.timeout) {
        options.timeout = 180;
    }
    if (!options.transition_to) {
        options.transition_to = 'stopped';
    }

    async.series([
        function (cb) {
            /* We load here to ensure this vm exists. */
            VM.load(uuid, function (err, obj) {
                if (err) {
                    VM.log.error(err);
                    cb(err);
                    return;
                } else {
                    vmobj = obj;
                    cb();
                }
            });
        }, function (cb) {
            // When stopping a VM that uses vm_autoboot, we assume we also do
            // the stop through vmadmd.
            if (BRAND_OPTIONS[vmobj.brand].features.use_vm_autoboot) {
                async.series([
                    function (callbk) {
                        setTransition(vmobj, 'stopping', options.transition_to,
                            (options.timeout * 1000), function (err) {

                            callbk(err);
                        });
                    }, function (callbk) {
                        postVmadmd(vmobj.uuid, 'stop',
                            {'timeout': options.timeout}, function (err) {

                            if (err) {
                                VM.log.error(err);
                            }
                            callbk(err);
                        });
                    }, function (callbk) {

                        // different version for VMs
                        unset_autoboot = 'select attr name=vm-autoboot; '
                            + 'set value=false; end';

                        zonecfg(['-u', uuid, unset_autoboot],
                            function (err, fds) {
                                if (err) {
                                    // The vm is dead at this point, failing
                                    // here would do no good, so we just log it.
                                    VM.log.error(err, 'stop(): Failed to '
                                        + unset_autoboot + ' for ' + uuid + ': '
                                        + err.message);
                                } else {
                                    VM.log.info('Stopped ' + uuid);
                                }
                                callbk();
                            }
                        );
                    }
                ], function (err) {
                    cb(err);
                });
            } else { // no vm_autoboot / vmadmd stop
                cb();
            }
        }, function (cb) {
            var args;

            // joyent brand specific stuff
            args = [vmobj.zonename, '/usr/sbin/shutdown', '-y', '-g', '0',
                '-i', '5'];

            // not using vm_autoboot means using the 'normal' boot process
            if (!BRAND_OPTIONS[vmobj.brand].features.use_vm_autoboot) {
                async.series([
                    function (callbk) {
                        VM.log.debug('/usr/sbin/zlogin ' + args.join(' '));
                        execFile('/usr/sbin/zlogin', args,
                            function (err, stdout, stderr) {

                            if (err) {
                                VM.log.error({'err': err, 'stdout': stdout,
                                    'stderr': stderr}, 'zlogin for '
                                    + vmobj.zonename + ' exited with code'
                                    + err.code + ': ' + err.message);
                                callbk(err);
                            } else {
                                VM.log.debug('zlogin claims to have worked, '
                                    + 'stdout:\n' + stdout + '\nstderr:\n'
                                    + stderr);
                                callbk();
                            }
                        });
                    }, function (callbk) {
                        zonecfg(['-u', uuid, unset_autoboot],
                            function (err, fds) {
                                if (err) {
                                    // The vm is dead at this point, failing
                                    // do no good, so we just log it.
                                    VM.log.warn(err, 'Failed to '
                                        + unset_autoboot + ' for ' + uuid + ': '
                                        + err.message);
                                } else {
                                    VM.log.info('Stopped ' + uuid);
                                }
                                callbk();
                            }
                        );
                    }
                ], function (err) {
                    cb(err);
                });
            } else { // using vmautoboot so won't shutdown from in the zone
                cb();
            }
        }, function (cb) {
            // Verify it's shut down
            VM.waitForZoneState(vmobj, 'installed', {}, function (err, result) {
                if (err) {
                    if (err.code === 'ETIMEOUT') {
                        VM.log.info('timeout waiting for zone to go to '
                            + '"installed"');
                    } else {
                        VM.log.error('unknown error waiting for zone to go'
                            + ' "installed"');
                    }
                    cb(err);
                } else {
                    // zone got to stopped
                    VM.log.info('VM seems to have switched to "installed"');
                    cb();
                }
            });
        }
    ], function (err) {
        callback(err);
    });
};

// sends several query-* commands to QMP to get details for a VM
exports.info = function (uuid, types, callback)
{
    ensureLogging('info', false);

    // load to ensure we're a VM
    VM.load(uuid, function (err, obj) {
        var type;

        if (err) {
            callback(err);
            return;
        }

        if (!BRAND_OPTIONS[obj.brand].features.runtime_info) {
            //  XXX if support is added to other brands, update this message.
            callback(new Error('the info command is only supported for KVM '
                + 'VMs'));
            return;
        }

        if (obj.state !== 'running' && obj.state !== 'stopping') {
            callback(new Error('Unable to get info for vm from state "'
                + obj.state + '", must be "running" or "stopping".'));
            return;
        }

        if (!types) {
            types = ['all'];
        }

        for (type in types) {
            type = types[type];
            if (VM.INFO_TYPES.indexOf(type) === -1) {
                callback(new Error('unknown info type: ' + type));
                return;
            }
        }

        http.get({ host: '127.0.0.1', port: 8080, path: '/vm/' + uuid + '/info'
            + '?types=' + types.join(',') }, function (res) {

                var data = '';

                if (res.statusCode !== 200) {
                    callback(new Error('Unable to get info from vmadmd, query '
                        + 'returned ' + res.statusCode + '.'));
                } else {
                    res.on('data', function (d) {
                        data = data + d.toString();
                    });
                    res.on('end', function (d) {
                        callback(null, JSON.parse(data));
                    });
                }
            }
        ).on('error', function (e) {
            console.error(e);
        });
    });
};

function reset(uuid, callback)
{
    VM.log.info('Resetting VM ' + uuid);

    /* We load here to ensure this vm exists. */
    VM.load(uuid, function (err, obj) {
        if (err) {
            callback(err);
            return;
        }

        if (obj.state !== 'running') {
            callback(new Error('Cannot reset vm from state "'
                + obj.state + '", must be "running".'));
            return;
        }

        if (BRAND_OPTIONS[obj.brand].features.use_vmadmd) {
            postVmadmd(obj.uuid, 'reset', {}, function (e) {
                if (e) {
                    callback(new Error('Unable to post "reset" to '
                        + 'vmadmd'));
                } else {
                    callback();
                }
            });
        } else {
            zoneadm(['-u', obj.uuid, 'reboot', '-X'], function (e, fds) {
                if (e) {
                    callback(new Error(rtrim(fds.stderr)));
                } else {
                    callback();
                }
            });
        }
    });
}

exports.reboot = function (uuid, options, callback)
{
    var m;

    if (options.hasOwnProperty('force') && options.force) {
        ensureLogging('reboot-F', true);
        reset(uuid, callback);
        return;
    } else {
        ensureLogging('reboot', true);
    }

    VM.log.info('Rebooting VM ' + uuid);

    if (!options) {
        options = {};
    }

    async.series([
        function (cb) {
            VM.load(uuid, function (err, vmobj) {
                if (err) {
                    cb(err);
                    return;
                }

                if (vmobj.state !== 'running') {
                    cb(new Error('Cannot reboot vm from state "' + vmobj.state
                        + '", must be "running"'));
                    return;
                }

                m = vmobj;
                cb();
            });
        }, function (cb) {
            // If nic tags have disappeared out from under us, don't allow a
            // reboot that will put us into a bad state
            lookupInvalidNicTags(m.nics, function (e) {
                if (e) {
                    cb(new Error('Cannot reboot vm: ' + e.message));
                    return;
                }

                cb();
            });
        }, function (cb) {
            var args;

            if (BRAND_OPTIONS[m.brand].features.use_vmadmd) {
                // here we stop the machine and set a transition so vmadmd will
                // start the machine once the stop finished.
                options.transition_to = 'start';
                VM.stop(uuid, options, function (err) {
                    if (err) {
                        cb(err);
                    } else {
                        cb();
                    }
                });
            } else {
                // joyent branded zones
                args = [m.zonename, '/usr/sbin/shutdown', '-y', '-g', '0',
                    '-i', '6'];
                VM.log.debug('/usr/sbin/zlogin ' + args.join(' '));
                execFile('/usr/sbin/zlogin', args,
                    function (err, stdout, stderr) {
                    if (err) {
                        VM.log.error({'err': err, 'stdout': stdout,
                            'stderr': stderr}, 'zlogin for ' + m.zonename
                            + ' exited with code' + err.code + ': '
                            + err.message);
                        cb(err);
                    } else {
                        cb();
                    }
                });
            }
        }
    ], function (err) {
        callback(err);
    });
};

exports.sysrq = function (uuid, req, options, callback)
{
    ensureLogging('sysrq-' + req, true);

    VM.log.info('Sending sysrq "' + req + '" to ' + uuid);

    /* We load here to ensure this vm exists. */
    VM.load(uuid, function (err, obj) {
        if (err) {
            callback(err);
            return;
        }

        if (obj.state !== 'running' && obj.state !== 'stopping') {
            callback(new Error('Unable to send request to vm from state "'
                + obj.state + '", must be "running" or "stopping".'));
            return;
        }

        if (BRAND_OPTIONS[obj.brand].features.type !== 'KVM') {
            callback(new Error('The sysrq command is only supported for KVM.'));
            return;
        }

        if (VM.SYSRQ_TYPES.indexOf(req) === -1) {
            callback(new Error('Invalid sysrq "' + req + '" valid values: '
                + '"' + VM.SYSRQ_TYPES.join('","') + '".'));
            return;
        }

        postVmadmd(obj.uuid, 'sysrq', {'request': req}, function (e) {
            if (e) {
                callback(new Error('Unable to post "sysrq" to vmadmd'));
            } else {
                callback();
            }
        });
    });
};

exports.console = function (uuid, callback)
{
    ensureLogging('console', false);

    VM.load(uuid, function (err, obj) {
        var args;
        var child;
        var cmd;
        var stty;

        if (err) {
            callback(err);
            return;
        }
        if (obj.state !== 'running') {
            callback(new Error('cannot connect to console when state is '
                + '"' + obj.state + '" must be "running".'));
            return;
        }

        if (BRAND_OPTIONS[obj.brand].features.zlogin_console) {
            cmd = '/usr/sbin/zlogin';
            args = ['-C', '-e', '\\035', obj.zonename];

            VM.log.debug(cmd + ' ' + args.join(' '));
            child = spawn(cmd, args, {customFds: [0, 1, 2]});
            child.on('close', function (code) {
                VM.log.debug('zlogin process exited with code ' + code);
                callback();
            });
        } else if (BRAND_OPTIONS[obj.brand].features.serial_console) {
            async.series([
                function (cb) {
                    cmd = '/usr/bin/stty';
                    args = ['-g'];
                    stty = '';

                    VM.log.debug(cmd + ' ' + args.join(' '));
                    child = spawn(cmd, args, {customFds: [0, -1, -1]});
                    child.stdout.on('data', function (data) {
                        // VM.log.debug('data: ' + data.toString());
                        stty = data.toString();
                    });
                    child.on('close', function (code) {
                        VM.log.debug('stty process exited with code '
                            + code);
                        cb();
                    });
                }, function (cb) {
                    cmd = '/usr/bin/socat';
                    args = ['unix-client:' + obj.zonepath
                        + '/root/tmp/vm.console', '-,raw,echo=0,escape=0x1d'];

                    VM.log.debug(cmd + ' ' + args.join(' '));
                    child = spawn(cmd, args, {customFds: [0, 1, 2]});
                    child.on('close', function (code) {
                        VM.log.debug('zlogin process exited with code '
                            + code);
                        cb();
                    });
                }, function (cb) {
                    cmd = '/usr/bin/stty';
                    args = [stty];

                    VM.log.debug(cmd + ' ' + args.join(' '));
                    child = spawn(cmd, args, {customFds: [0, -1, -1]});
                    child.on('close', function (code) {
                        VM.log.debug('stty process exited with code '
                            + code);
                        cb();
                    });
                }
            ], function (e, results) {
                callback(e);
            });
        } else {
            callback(new Error('Cannot get console for brand: ' + obj.brand));
        }
    });
};