Skip to content
This repository
Browse code

OS-1027 add 'provisioning' state for VMs when they're created until t…

…hey're actually 'running'.
  • Loading branch information...
commit 1663802a8320e5f3eb9a3a8d27cdb5ca7698b229 1 parent 942fddb
joshwilsdon authored November 29, 2012
25  overlay/generic/lib/svc/method/mdata-execute
@@ -29,23 +29,16 @@ set -o xtrace
29 29
 . /lib/svc/share/smf_include.sh
30 30
 smf_is_globalzone && exit ${SMF_EXIT_OK}
31 31
 
32  
-  user_script_exit=${SMF_EXIT_OK}
33  
-  if [ -x /var/svc/mdata-user-script ]; then
  32
+# If we got as far as running the user-script the 'provision' was a success
  33
+# from here out a failure will leave the zone running.
  34
+if [ -f /var/svc/provisioning ]; then
  35
+    mv /var/svc/provision{ing,_success}
  36
+fi
  37
+
  38
+user_script_exit=${SMF_EXIT_OK}
  39
+if [ -x /var/svc/mdata-user-script ]; then
34 40
     /var/svc/mdata-user-script
35 41
     [ $? -gt 0 ] && user_script_exit=${SMF_EXIT_ERR_FATAL}
36  
-  fi
37  
-
38  
-  # Finish the provisioning process (if the case)
39  
-
40  
-  if [ -f /var/svc/provisioning ]; then
41  
-    if [ ${user_script_exit} -eq 0 ]; then
42  
-      # If 'provision_ongoing' exists, we assume the user wants to control
43  
-      # the final 'success' flag himself and will rename the 'provisioning'
44  
-      # file as appropriate. Otherwise, announce success ourselves.
45  
-      [ -f /var/svc/provision_ongoing ] || mv /var/svc/provision{ing,_success}
46  
-    else
47  
-      mv /var/svc/provision{ing,_failure}
48  
-    fi
49  
-  fi
  42
+fi
50 43
 
51 44
 exit ${user_script_exit}
34  overlay/generic/usr/lib/brand/kvm/kinstall
@@ -76,31 +76,37 @@ PDS_NAME=`mount | nawk -v p=$dname '{if ($1 == p) print $3}'`
76 76
 # it's possible to specify a zone root here if you specified the
77 77
 # '-x nodataset' when installing the zone.
78 78
 if [[ -n ${TMPLZONE} ]]; then
79  
-    QUOTA_ARG=
80  
-    if [[ ${ZQUOTA} != "0" ]]; then
81  
-        QUOTA_ARG="-o quota=${ZQUOTA}g"
82  
-    fi
  79
+	QUOTA_ARG=
  80
+	if [[ ${ZQUOTA} != "0" ]]; then
  81
+		QUOTA_ARG="-o quota=${ZQUOTA}g"
  82
+	fi
83 83
 
84  
-    zfs snapshot $PDS_NAME/${TMPLZONE}@${bname}
85  
-    zfs clone ${QUOTA_ARG} $PDS_NAME/${TMPLZONE}@${bname} $PDS_NAME/$bname
  84
+	zfs snapshot $PDS_NAME/${TMPLZONE}@${bname}
  85
+	zfs clone ${QUOTA_ARG} $PDS_NAME/${TMPLZONE}@${bname} $PDS_NAME/$bname
86 86
 elif [[ ${ZQUOTA} != "0" ]]; then
87  
-    # don't have a template dataset, so we set the quota on the fresh zoneroot
88  
-    zfs set quota=${ZQUOTA}g ${PDS_NAME}/${bname}
  87
+	# don't have a template dataset, so we set the quota on the fresh zoneroot
  88
+	zfs set quota=${ZQUOTA}g ${PDS_NAME}/${bname}
89 89
 fi
90 90
 
91 91
 if [ ! -d ${ZONEPATH}/config ]; then
92  
-    mkdir -p ${ZONEPATH}/config
93  
-    chmod 755 ${ZONEPATH}/config
  92
+	mkdir -p ${ZONEPATH}/config
  93
+	chmod 755 ${ZONEPATH}/config
94 94
 fi
95 95
 
96 96
 if [ ! -d ${ZROOT} ]; then
97  
-    mkdir -p ${ZROOT}
98  
-    chmod 755 ${ZROOT}
  97
+	mkdir -p ${ZROOT}
  98
+	chmod 755 ${ZROOT}
99 99
 fi
100 100
 
101 101
 if [ ! -d ${ZROOT}/tmp ]; then
102  
-    mkdir -p ${ZROOT}/tmp
103  
-    chmod 1777 ${ZROOT}/tmp
  102
+	mkdir -p ${ZROOT}/tmp
  103
+	chmod 1777 ${ZROOT}/tmp
  104
+fi
  105
+
  106
+# make /var/svc for the 'provisioning file'
  107
+if [ ! -d ${ZROOT}/var/svc ]; then
  108
+	mkdir -p ${ZROOT}/var/svc
  109
+	chmod 0755 ${ZROOT}/var/svc
104 110
 fi
105 111
 
106 112
 # The dataset quota must be a number.
12  src/vm/common/vmtest.js
@@ -173,6 +173,8 @@ exports.on_new_vm = function(t, uuid, payload, state, fnlist, callback)
173 173
     });
174 174
 
175 175
     async.series(functions, function (err) {
  176
+        var openThingies;
  177
+
176 178
         if (err) {
177 179
             t.ok(false, err.message);
178 180
         }
@@ -180,7 +182,15 @@ exports.on_new_vm = function(t, uuid, payload, state, fnlist, callback)
180 182
             // up to caller to call t.end!
181 183
             return callback();
182 184
         } else {
183  
-             t.end();
  185
+            t.end();
  186
+
  187
+            /*
  188
+
  189
+            // Helpful bit from Isaac that tells what's still open.
  190
+            openThingies = process._getActiveHandles();
  191
+            console.dir(openThingies);
  192
+
  193
+            */
184 194
         }
185 195
     });
186 196
 };
22  src/vm/man/vmadm.1m.md
Source Rendered
@@ -913,6 +913,22 @@ tab-complete UUIDs rather than having to type them out for every command.
913 913
         update: yes (live update)
914 914
         default: value of max_physical_memory
915 915
 
  916
+    mdata_exec_timeout:
  917
+
  918
+        For OS VMs this parameter adjusts the timeout on the start method of
  919
+        the svc:/smartdc/mdata:execute service running in the zone. This is the
  920
+        service which runs user-script scripts.
  921
+
  922
+        This parameter only makes sense when creating a VM and is ignored
  923
+        in other cases.
  924
+
  925
+        type: integer (0 for unlimited, >0 number of seconds)
  926
+        vmtype: OS
  927
+        listable: no
  928
+        create: yes
  929
+        update: no
  930
+        default: 300
  931
+
916 932
     nics:
917 933
 
918 934
         When creating a KVM VM or getting a KVM VM's JSON, you will use this
@@ -1373,9 +1389,11 @@ tab-complete UUIDs rather than having to type them out for every command.
1373 1389
         When a KVM VM is in transition from running to either 'off' (in the
1374 1390
         case of stop) or 'start' (in the case of reboot), the transition_to
1375 1391
         field will be set to indicate which state the VM is transitioning to.
  1392
+        Additionally when a VM is provisioning you may see this with a value
  1393
+        of 'running'.
1376 1394
 
1377  
-        type: string value, one of: ['stopped', 'start']
1378  
-        vmtype: KVM
  1395
+        type: string value, one of: ['stopped', 'start', 'running']
  1396
+        vmtype: OS,KVM
1379 1397
         listable: no
1380 1398
         create: no
1381 1399
         update: no
922  src/vm/node_modules/VM.js
@@ -22,7 +22,8 @@
22 22
  *
23 23
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
24 24
  *
25  
- * Experimental functions, expect these interfaces to be unstable:
  25
+ * Experimental functions, expect these interfaces to be unstable and
  26
+ * potentially go away entirely:
26 27
  *
27 28
  * install(uuid, callback)
28 29
  * receive(target, options, callback)
@@ -70,6 +71,7 @@ var fs = require('fs');
70 71
 var http = require('http');
71 72
 var net = require('net');
72 73
 var path = require('path');
  74
+var Qmp = require('/usr/vm/node_modules/qmp').Qmp;
73 75
 var spawn = cp.spawn;
74 76
 var sprintf = require('/usr/node/node_modules/sprintf').sprintf;
75 77
 var system = require('/usr/node/node_modules/system');
@@ -105,8 +107,11 @@ exports.FLATTENABLE_HASH_KEYS = [
105 107
     'tags'
106 108
 ];
107 109
 
108  
-var VM = this;
  110
+var DEFAULT_MDATA_TIMEOUT = 300;
  111
+var PROVISION_TIMEOUT = 300;
109 112
 var STOP_TIMEOUT = 60;
  113
+var VM = this;
  114
+
110 115
 VM.log = null;
111 116
 
112 117
 // can be (re)set by loader before we start.
@@ -121,8 +126,14 @@ exports.loglevel = 'debug';
121 126
 function OpenOnErrorFileStream(filename) {
122 127
     this.path = filename;
123 128
     this.write = this.constructor.prototype.write1;
  129
+    this.end = this.constructor.prototype.end1;
124 130
 }
125 131
 
  132
+OpenOnErrorFileStream.prototype.end1 = function () {
  133
+    // in initial mode we're not writing anything, so nothing to flush
  134
+    return;
  135
+};
  136
+
126 137
 // used until first ERROR or higher, then opens file and ensures future writes
127 138
 // go to .write2()
128 139
 OpenOnErrorFileStream.prototype.write1 = function (rec) {
@@ -131,6 +142,7 @@ OpenOnErrorFileStream.prototype.write1 = function (rec) {
131 142
     if (rec.level >= bunyan.ERROR || log_to_file) {
132 143
         this.stream = fs.createWriteStream(this.path,
133 144
             {flags: 'a', encoding: 'utf8'});
  145
+        this.end = this.stream.end;
134 146
         this.write = this.constructor.prototype.write2;
135 147
         // dump out logs from ringbuffer too since there was an error so we can
136 148
         // figure out what's going on.
@@ -336,6 +348,7 @@ exports.INFO_TYPES = [
336 348
     'kvm',
337 349
     'pci',
338 350
     'spice',
  351
+    'status',
339 352
     'version',
340 353
     'vnc'
341 354
 ];
@@ -384,6 +397,7 @@ var XML_PROPERTIES = {
384 397
         'default-gateway': 'default_gateway',
385 398
         'dns-domain': 'dns_domain',
386 399
         'do-not-inventory': 'do_not_inventory',
  400
+        'failed': 'failed',
387 401
         'hostname': 'hostname',
388 402
         'never-booted': 'never_booted',
389 403
         'owner-uuid': 'owner_uuid',
@@ -576,6 +590,7 @@ var PAYLOAD_PROPERTIES = {
576 590
     'max_lwps': {'type': 'integer'},
577 591
     'max_physical_memory': {'type': 'integer'},
578 592
     'max_swap': {'type': 'integer'},
  593
+    'mdata_exec_timeout': {'type': 'integer'},
579 594
     'nics': {'type': 'object-array'},
580 595
     'nics.*.allow_dhcp_spoofing': {'type': 'boolean'},
581 596
     'nics.*.allow_ip_spoofing': {'type': 'boolean'},
@@ -670,6 +685,7 @@ var joyent_allowed = {
670 685
     'max_lwps': ['create', 'receive', 'update'],
671 686
     'max_physical_memory': ['create', 'receive', 'update'],
672 687
     'max_swap': ['create', 'receive', 'update'],
  688
+    'mdata_exec_timeout': ['create'],
673 689
     'nics': ['create', 'receive'],
674 690
     'nics.*.allow_dhcp_spoofing': ['add', 'update'],
675 691
     'nics.*.allow_ip_spoofing': ['add', 'update'],
@@ -735,14 +751,16 @@ var joyent_allowed = {
735 751
  * 'runtime_info' -- (boolean) whether this zone supports the 'info' command
736 752
  * 'serial_console' -- (boolean) whether this zone uses serial console
737 753
  * 'type' -- the type of the VM (OS or KVM), all brands should include this
  754
+ * 'update_mdata_exec_timeout' (boolean) whether to update mdata:exec timeout
738 755
  * 'update_rctls' (boolean) whether we can update rctls 'live' for this zone
739 756
  * 'use_tmpfs' -- (boolean) whether this type of zone uses tmpfs
740 757
  * 'use_vm_autoboot' -- (boolean) use vm-autoboot instead of autoboot
741 758
  * 'use_vmadmd' -- (boolean) use vmadmd for some actions instead of direct
742  
- * 'wait_for_reboot' -- (boolean) whether we should wait for a reboot on create
743  
- * 'write_zoneconfig' -- (boolean) whether we should create a zoneinit config
  759
+ * 'var_svc_provisioning' -- (boolean) whether brand uses /var/svc/provisioning
  760
+ * 'wait_for_hwsetup' -- (boolean) use QMP and provision_success when hwsetup
744 761
  * 'write_zone_netfiles' -- (boolean) write out files like /etc/hostname.net0
745 762
  * 'zlogin_console' -- (boolean) use zlogin -C for console (vs. serial_console)
  763
+ * 'zoneinit' -- (boolean) this brand's setup may be controlled by zoneinit
746 764
  *
747 765
  * All of the keys:
748 766
  *
@@ -759,13 +777,14 @@ var BRAND_OPTIONS = {
759 777
             'brand': ['create', 'receive'],
760 778
             'image_uuid': ['create', 'receive']
761 779
         }, 'features': {
  780
+            'cleanup_dataset': true,
762 781
             'type': 'OS',
  782
+            'update_mdata_exec_timeout': true,
763 783
             'update_rctls': true,
764 784
             'use_tmpfs': true,
765  
-            'wait_for_reboot': true,
766  
-            'write_zoneconfig': true,
767 785
             'write_zone_netfiles': true,
768  
-            'zlogin_console': true
  786
+            'zlogin_console': true,
  787
+            'zoneinit': true
769 788
         }
770 789
     }, 'joyent-minimal': {
771 790
         'allowed_properties': joyent_allowed,
@@ -775,8 +794,10 @@ var BRAND_OPTIONS = {
775 794
         }, 'features': {
776 795
             'cleanup_dataset': true,
777 796
             'type': 'OS',
  797
+            'update_mdata_exec_timeout': true,
778 798
             'update_rctls': true,
779 799
             'use_tmpfs': true,
  800
+            'var_svc_provisioning': true,
780 801
             'write_zone_netfiles': true,
781 802
             'zlogin_console': true
782 803
         }
@@ -873,18 +894,20 @@ var BRAND_OPTIONS = {
873 894
         }, 'required_properties': {
874 895
             'brand': ['create', 'receive']
875 896
         }, 'features': {
876  
-            'type': 'KVM',
877  
-            'use_vm_autoboot': true,
878  
-            'use_vmadmd': true,
879  
-            'pid_file': '/tmp/vm.pid',
880 897
             'check_avail_ram': true,
881  
-            'model_required': true,
882  
-            'serial_console': true,
883  
-            'runtime_info': true,
884 898
             'default_memory_overhead': VM.KVM_MEM_OVERHEAD,
885  
-            'min_memory_overhead': VM.KVM_MIN_MEM_OVERHEAD,
886 899
             'limit_priv': ['default', '-file_link_any', '-net_access',
887  
-                '-proc_fork', '-proc_info', '-proc_session']
  900
+                '-proc_fork', '-proc_info', '-proc_session'],
  901
+            'min_memory_overhead': VM.KVM_MIN_MEM_OVERHEAD,
  902
+            'model_required': true,
  903
+            'pid_file': '/tmp/vm.pid',
  904
+            'runtime_info': true,
  905
+            'serial_console': true,
  906
+            'type': 'KVM',
  907
+            'use_vm_autoboot': true,
  908
+            'use_vmadmd': true,
  909
+            'var_svc_provisioning': true,
  910
+            'wait_for_hwsetup': true
888 911
         }
889 912
     }
890 913
 };
@@ -1777,7 +1800,7 @@ function preloadZoneData(uuid, callback)
1777 1800
                 var z = data.records[z_uuid];
1778 1801
 
1779 1802
                 // NOTE: z.state here is equivalent to zone_state not state.
1780  
-                if (BRAND_OPTIONS[z.brand].hasOwnProperty('features')
  1803
+                if (z && BRAND_OPTIONS[z.brand].hasOwnProperty('features')
1781 1804
                     && BRAND_OPTIONS[z.brand].features.pid_file
1782 1805
                     && z.state === 'running') {
1783 1806
 
@@ -1814,7 +1837,7 @@ function preloadZoneData(uuid, callback)
1814 1837
             });
1815 1838
         }
1816 1839
     ], function (err, res) {
1817  
-        VM.log.debug('leaving preloadZoneData()');
  1840
+        VM.log.trace('leaving preloadZoneData()');
1818 1841
         callback(err, data);
1819 1842
     });
1820 1843
 }
@@ -1854,8 +1877,8 @@ function getZoneRecords(uuid, callback)
1854 1877
             if (!errmsg.match(/No such zone configured$/)) {
1855 1878
                 // not existing isn't always a problem (eg. existence check), so
1856 1879
                 // don't always log it as an error.
1857  
-                VM.log.error('getZoneRecords() zoneadm "'
1858  
-                    + args.join(',') + '" failed', err);
  1880
+                VM.log.error(err, 'getZoneRecords() zoneadm "'
  1881
+                    + args.join(',') + '" failed');
1859 1882
             }
1860 1883
 
1861 1884
             callback(new Error(errmsg));
@@ -1929,13 +1952,13 @@ function getLastModified(vmobj)
1929 1952
         files.push(path.join(vmobj.zonepath, '/config/metadata.json'));
1930 1953
         files.push(path.join(vmobj.zonepath, '/config/tags.json'));
1931 1954
     } else {
1932  
-        VM.log.debug('getLastModified() no zonepath!\n');
  1955
+        VM.log.debug('getLastModified() no zonepath!');
1933 1956
     }
1934 1957
 
1935 1958
     if (vmobj.hasOwnProperty('zonename')) {
1936 1959
         files.push('/etc/zones/' + vmobj.zonename + '.xml');
1937 1960
     } else {
1938  
-        VM.log.debug('getLastModified() no zonename!\n');
  1961
+        VM.log.debug('getLastModified() no zonename!');
1939 1962
     }
1940 1963
 
1941 1964
     for (file in files) {
@@ -1949,7 +1972,7 @@ function getLastModified(vmobj)
1949 1972
             }
1950 1973
         } catch (e) {
1951 1974
             if (e.code !== 'ENOENT') {
1952  
-                VM.log.error('Unable to get timestamp for "' + file + '":'
  1975
+                VM.log.error(e, 'Unable to get timestamp for "' + file + '":'
1953 1976
                     + e.message);
1954 1977
             }
1955 1978
         }
@@ -1969,7 +1992,7 @@ function loadVM(uuid, data, callback)
1969 1992
     if (!info) {
1970 1993
         e = new Error('VM.load() empty info when getting record '
1971 1994
             + 'for vm ' + uuid);
1972  
-        VM.log.error(e.message, e);
  1995
+        VM.log.error(e);
1973 1996
         callback(e);
1974 1997
         return;
1975 1998
     }
@@ -2016,13 +2039,21 @@ function loadVM(uuid, data, callback)
2016 2039
             vmobj.pid = info.pid;
2017 2040
         }
2018 2041
 
2019  
-        // state could already be set here if it was overriden by a
2020  
-        // transition that's in progress.
  2042
+        // state could already be set here if it was overriden by a transition
  2043
+        // that's in progress. So we only change if that's not the case.
2021 2044
         vmobj.zone_state = info.state;
2022  
-        if (info.state === 'installed') {
2023  
-            vmobj.state = 'stopped';
2024  
-        } else if (!vmobj.hasOwnProperty('state')) {
2025  
-            vmobj.state = info.state;
  2045
+        if (!vmobj.hasOwnProperty('state')) {
  2046
+            if (info.state === 'installed') {
  2047
+                vmobj.state = 'stopped';
  2048
+            } else {
  2049
+                vmobj.state = info.state;
  2050
+            }
  2051
+        }
  2052
+
  2053
+        // If the zone has the 'failed' property it doesn't matter what
  2054
+        // other state it might be in, we list its state as 'failed'.
  2055
+        if (vmobj.failed) {
  2056
+            vmobj.state = 'failed';
2026 2057
         }
2027 2058
 
2028 2059
         loadMetadata(vmobj, function (error, metadata, tags) {
@@ -2041,7 +2072,7 @@ function loadVM(uuid, data, callback)
2041 2072
             // when 'incomplete' because of this.
2042 2073
             if (error) {
2043 2074
                 if (vmobj.zone_state !== 'incomplete') {
2044  
-                    VM.log.debug('zone is in state incomplete ignoring '
  2075
+                    VM.log.debug(error, 'zone is in state incomplete ignoring '
2045 2076
                         + 'error: ' + error.message);
2046 2077
                 } else {
2047 2078
                     callback(new Error('Unable to add metadata:'
@@ -2206,8 +2237,8 @@ exports.load = function (uuid, callback)
2206 2237
 
2207 2238
     preloadZoneData(uuid, function (error, data) {
2208 2239
         if (error) {
2209  
-            VM.log.error('VM.load() failed to get zone record'
2210  
-                + ' for ' + uuid, error);
  2240
+            VM.log.error(error, 'VM.load() failed to get zone record'
  2241
+                + ' for ' + uuid);
2211 2242
             callback(error);
2212 2243
         } else {
2213 2244
             loadVM(uuid, data, callback);
@@ -2419,8 +2450,8 @@ exports.lookup = function (search, options, callback) {
2419 2450
                 // create all the volumes we found that we need.
2420 2451
                 async.forEachSeries(results, expander, function (e) {
2421 2452
                     if (e) {
2422  
-                        VM.log.error('VM.lookup failed to '
2423  
-                            + 'expand results', e);
  2453
+                        VM.log.error(e, 'VM.lookup failed to expand results: '
  2454
+                            + e.message);
2424 2455
                         callback(e);
2425 2456
                     } else {
2426 2457
                         callback(null, full_results);
@@ -2525,9 +2556,9 @@ function checkDatasets(payload, callback)
2525 2556
     function checker(dataset, cb) {
2526 2557
         zfs(['list', '-o', 'name', '-H', dataset], function (err, fds) {
2527 2558
             if (err) {
2528  
-                VM.log.error('zfs list ' + dataset + ' exited with'
2529  
-                    + ' code: ' + err.code + ' stdout: "' + fds.stdout
2530  
-                    + '" stderr:"' + fds.stderr + '"', err);
  2559
+                VM.log.error({'err': err, 'stdout': fds.stdout,
  2560
+                    'stderr': fds.stderr}, 'zfs list ' + dataset + ' '
  2561
+                    + 'exited with' + ' code ' + err.code + ': ' + err.message);
2531 2562
                 cb(new Error('unable to find dataset: ' + dataset));
2532 2563
             } else {
2533 2564
                 cb();
@@ -2538,8 +2569,8 @@ function checkDatasets(payload, callback)
2538 2569
     // check that we have all the volumes
2539 2570
     async.forEachSeries(checkme, checker, function (err) {
2540 2571
         if (err) {
2541  
-            VM.log.error('checkDatasets() failed to find required '
2542  
-                + 'volumes', err);
  2572
+            VM.log.error(err, 'checkDatasets() failed to find required '
  2573
+                + 'volumes');
2543 2574
             callback(err);
2544 2575
         } else {
2545 2576
             // progress(100, 'we have all necessary datasets');
@@ -2981,21 +3012,29 @@ function createVM(payload, callback)
2981 3012
     });
2982 3013
 }
2983 3014
 
2984  
-function cleanupMessyDataset(zonepath, callback)
  3015
+function cleanupMessyDataset(zonepath, brand, callback)
2985 3016
 {
2986 3017
     var command;
2987 3018
     var zoneroot = path.join(zonepath, '/root');
2988 3019
 
2989  
-    // Some datasets leave junk behind that they shouldn't. This cleans
2990  
-    // up junk files to ensure the zone comes up a bit cleaner.
  3020
+    // Some datasets leave junk behind that they shouldn't or is only there for
  3021
+    // old platforms. This cleans up junk files to ensure the zone comes up a
  3022
+    // bit cleaner.
2991 3023
     fs.exists(zoneroot, function (exists) {
2992 3024
         if (exists) {
2993 3025
             command = 'rm -rf '
2994 3026
                 + zoneroot + '/var/adm/utmpx '
2995 3027
                 + zoneroot + '/var/adm/wtmpx '
2996 3028
                 + zoneroot + '/var/svc/log/*.log '
2997  
-                + zoneroot + '/root/zone* '
2998  
-                + '&& touch ' + zoneroot + '/var/adm/wtmpx';
  3029
+                + zoneroot + '/var/svc/mdata '
  3030
+                + zoneroot + '/var/svc/manifest/mdata.xml ';
  3031
+
  3032
+            if (! BRAND_OPTIONS[brand].features.zoneinit) {
  3033
+                // eg. joyent-minimal (don't need zoneinit)
  3034
+                command = command + zoneroot + '/root/zone* ';
  3035
+            }
  3036
+
  3037
+            command = command + '&& touch ' + zoneroot + '/var/adm/wtmpx';
2999 3038
             VM.log.debug(command);
3000 3039
             exec(command, function (error, stdout, stderr) {
3001 3040
                 VM.log.debug('stdout: ' + stdout);
@@ -3247,7 +3286,7 @@ exports.getSysinfo = function (args, callback)
3247 3286
 exports.waitForZoneState = function (payload, state, options, callback) {
3248 3287
     var sysevent_state;
3249 3288
     var timeout;
3250  
-    var timeout_secs = 5 * 60;
  3289
+    var timeout_secs = PROVISION_TIMEOUT;
3251 3290
     var watcher;
3252 3291
 
3253 3292
     ensureLogging('waitForZoneState', false);
@@ -3878,12 +3917,17 @@ function checkPayloadProperties(payload, vmobj, callback)
3878 3917
         // can't update disks of a running VM
3879 3918
         if ((payload.hasOwnProperty('add_disks')
3880 3919
             || payload.hasOwnProperty('remove_disks')
3881  
-            || payload.hasOwnProperty('update_disks'))
3882  
-            && vmobj.state !== 'stopped') {
  3920
+            || payload.hasOwnProperty('update_disks'))) {
3883 3921
 
3884  
-            callback(new Error('updates to disks are only allowed when state '
3885  
-                + 'is "stopped", currently: ' + vmobj.state));
3886  
-            return;
  3922
+            if ((vmobj.state !== 'stopped')
  3923
+                || (vmobj.state === 'provisioning'
  3924
+                && vmobj.zone_state !== 'installed')) {
  3925
+
  3926
+                callback(new Error('updates to disks are only allowed when '
  3927
+                    + 'state is "stopped", currently: ' + vmobj.state + ' ('
  3928
+                    + vmobj.zonestate + ')'));
  3929
+                return;
  3930
+            }
3887 3931
         }
3888 3932
 
3889 3933
         // if there's a min_overhead we ensure values are higher than ram.
@@ -4317,8 +4361,8 @@ function createDelegatedDataset(payload, callback)
4317 4361
             zcfg = zcfg + 'add dataset; set name=' + ds + '; end\n';
4318 4362
             zonecfg(['-u', payload.uuid, zcfg], function (e, fds) {
4319 4363
                 if (e) {
4320  
-                    VM.log.error('unable to add delegated dataset to '
4321  
-                        + payload.uuid + ' stderr: ' + fds.stderr, e);
  4364
+                    VM.log.error({'err': e, 'stderr': fds.stderr},
  4365
+                        'unable to add delegated dataset to ' + payload.uuid);
4322 4366
                     callback(e);
4323 4367
                 } else {
4324 4368
                     callback();
@@ -4901,11 +4945,372 @@ function buildZonecfgUpdate(vmobj, payload)
4901 4945
     return zcfg;
4902 4946
 }
4903 4947
 
  4948
+// Checks that QMP is responding to query-status and if so passes the boolean
  4949
+// value of the hwsetup parameter to the callback.
  4950
+function checkHWSetup(vmobj, callback)
  4951
+{
  4952
+    var q;
  4953
+    var socket;
  4954
+
  4955
+    q = new Qmp(VM.log);
  4956
+    socket = vmobj.zonepath + '/root/tmp/vm.qmp';
  4957
+
  4958
+    q.connect(socket, function (error) {
  4959
+        if (error) {
  4960
+            VM.log.error(error, 'q.connect(): Error: ' + error.message);
  4961
+            callback(error);
  4962
+            return;
  4963
+        }
  4964
+        q.command('query-status', null, function (e, result) {
  4965
+            if (e) {
  4966
+                VM.log.error(e, 'q.command(query-status): Error: ' + e.message);
  4967
+                callback(e);
  4968
+                return;
  4969
+            }
  4970
+            q.disconnect();
  4971
+            callback(null, result.hwsetup ? true : false);
  4972
+            return;
  4973
+        });
  4974
+    });
  4975
+}
  4976
+
  4977
+// cb (if set) will be called with an Error if we can't setup the interval loop
  4978
+// otherwise when the loop is shut down.
  4979
+function markProvisionedWhenHWSetup(vmobj, options, cb)
  4980
+{
  4981
+    var loop_interval = 3; // seconds
  4982
+    var ival_handle;
  4983
+
  4984
+    if (!cb) {
  4985
+        // if no cb() is passed in just log if there was an error
  4986
+        cb = function (err) {
  4987
+            if (err) {
  4988
+                VM.log.warn(err, 'cb() not passed, ignoring error: '
  4989
+                    + err.message);
  4990
+            }
  4991
+        };
  4992
+    }
  4993
+
  4994
+    if (!BRAND_OPTIONS[vmobj.brand].features.wait_for_hwsetup) {
  4995
+        // do nothing for zones where we don't wait for hwsetup
  4996
+        cb(new Error('brand ' + vmobj.brand + ' does not support hwsetup'));
  4997
+        return;
  4998
+    }
  4999
+
  5000
+    if (!options) {
  5001
+        options = {};
  5002
+    }
  5003
+
  5004
+    // if caller wants they can change the interval
  5005
+    if (options.hasOwnProperty('interval')) {
  5006
+        loop_interval = options.interval;
  5007
+    }
  5008
+
  5009
+    ival_handle = setInterval(function () {
  5010
+        VM.load(vmobj.uuid, function (err, obj) {
  5011
+            var timeout_remaining;
  5012
+
  5013
+            function done() {
  5014
+                if (ival_handle) {
  5015
+                    clearInterval(ival_handle);
  5016
+                    ival_handle = null;
  5017
+                }
  5018
+            }
  5019
+
  5020
+            if (err) {
  5021
+                // If the VM was deleted between calls, nothing much we can do.
  5022
+                VM.log.error(err, 'Unable to load ' + vmobj.uuid + ' '
  5023
+                    + err.message);
  5024
+                cb(err);
  5025
+                done();
  5026
+                return;
  5027
+            }
  5028
+
  5029
+            // we only do anything if we're still waiting for provisioning
  5030
+            if (vmobj.state !== 'provisioning') {
  5031
+                done();
  5032
+                cb();
  5033
+                return;
  5034
+            }
  5035
+
  5036
+            timeout_remaining =
  5037
+                (Number(obj.transition_expire) - Date.now(0)) / 1000;
  5038
+
  5039
+            if (timeout_remaining <= 0) {
  5040
+                // IMPORTANT: this may run multiple times, must be idempotent
  5041
+
  5042
+                markVMFailure(vmobj, function (mark_err) {
  5043
+                    VM.log.warn(mark_err, 'zoneinit failed, zone is '
  5044
+                        + 'being stopped for manual investigation.');
  5045
+                    cb();
  5046
+                    done();
  5047
+                });
  5048
+                return;
  5049
+            }
  5050
+
  5051
+            checkHWSetup(vmobj, function (check_err, result) {
  5052
+                if (check_err) {
  5053
+                    VM.log.debug(check_err, 'checkHWSetup Error: '
  5054
+                        + check_err.message);
  5055
+                    return;
  5056
+                }
  5057
+
  5058
+                if (result) {
  5059
+                    VM.log.debug('QMP says VM ' + vmobj.uuid
  5060
+                        + ' completed hwsetup');
  5061
+                    unsetTransition(vmobj, function (unset_err) {
  5062
+                        var provisioning;
  5063
+                        var provision_success;
  5064
+
  5065
+                        provisioning = path.join(vmobj.zonepath,
  5066
+                            '/root/var/svc/provisioning');
  5067
+                        provision_success = path.join(vmobj.zonepath,
  5068
+                            '/root/var/svc/provision_success');
  5069
+
  5070
+                        if (unset_err) {
  5071
+                            VM.log.error(unset_err);
  5072
+                        } else {
  5073
+                            VM.log.debug('cleared transition to provisioning on'
  5074
+                                + ' ' + vmobj.uuid);
  5075
+                        }
  5076
+
  5077
+                        fs.rename(provisioning, provision_success,
  5078
+                            function (e) {
  5079
+
  5080
+                            if (e) {
  5081
+                                if (e.code !== 'ENOENT') {
  5082
+                                    VM.log.error(e);
  5083
+                                    cb(err);
  5084
+                                } else {
  5085
+                                    VM.log.debug(e);
  5086
+                                    // in this case the file doesn't exist so
  5087
+                                    // either moved it already from outside or
  5088
+                                    // deleted the machine, either way the next
  5089
+                                    // run of the Interval loop should fix.
  5090
+                                }
  5091
+                                return;
  5092
+                            }
  5093
+
  5094
+                            cb();
  5095
+                            done();
  5096
+                        });
  5097
+                    });
  5098
+                }
  5099
+            });
  5100
+        });
  5101
+    }, loop_interval * 1000);
  5102
+}
  5103
+
  5104
+// m argument should have zonename and uuid members.
  5105
+function markVMFailure(m, cb)
  5106
+{
  5107
+    if (!m || !m.hasOwnProperty('uuid') || !m.hasOwnProperty('zonename')) {
  5108
+        cb(new Error('markVMFailure needs uuid + zonename'));
  5109
+        return;
  5110
+    }
  5111
+
  5112
+    execFile('/usr/bin/ptree', ['-z', m.zonename],
  5113
+        function (ptree_err, stdout, stderr) {
  5114
+            var zcfg;
  5115
+
  5116
+            if (ptree_err) {
  5117
+                VM.log.error(ptree_err, 'unable to get ptree from ' + m.uuid
  5118
+                    + ': ' + stderr);
  5119
+            } else {
  5120
+                VM.log.warn('processes running in ' + m.uuid
  5121
+                    + ' at fail time:\n' + stdout);
  5122
+            }
  5123
+            VM.log.warn(ptree_err, 'zone setup failed, zone is being stopped '
  5124
+                + 'for manual investigation.');
  5125
+
  5126
+            // Mark the zone as 'failed'
  5127
+            zcfg = 'add attr; set name=failed; set value="provisioning"; '
  5128
+                + 'set type=string; end';
  5129
+
  5130
+            zonecfg(['-u', m.uuid, zcfg], function (zonecfg_err) {
  5131
+
  5132
+                if (zonecfg_err) {
  5133
+                    VM.log.error(zonecfg_err, 'Unable to set failure flag on '
  5134
+                        + m.uuid + ': ' + zonecfg_err.message);
  5135
+                } else {
  5136
+                    VM.log.debug('set failure flag on ' + m.uuid);
  5137
+                }
  5138
+
  5139
+                // attempt to remove transition
  5140
+                unsetTransition(m, function (unset_err) {
  5141
+                    if (unset_err) {
  5142
+                        VM.log.error(unset_err);
  5143
+                    }
  5144
+
  5145
+                    VM.stop(m.uuid, {'force': true}, function (stop_err) {
  5146
+                        // only log errors because there's nothing to do
  5147
+
  5148
+                        if (stop_err) {
  5149
+                            VM.log.error(stop_err, 'failed to stop VM ' + m.uuid
  5150
+                                + ': ' + stop_err.message);
  5151
+                        }
  5152
+
  5153
+                        cb();
  5154
+                    });
  5155
+                });
  5156
+            });
  5157
+        }
  5158
+    );
  5159
+}
  5160
+
  5161
+function waitForZoneinit(payload, cb)
  5162
+{
  5163
+    VM.waitForZoneState(payload, 'running', {}, function (error) {
  5164
+        if (error) {
  5165
+            markVMFailure(payload, function (err) {
  5166
+                VM.log.warn(err, 'zoneinit failed, zone is being '
  5167
+                    + 'stopped for manual investigation.');
  5168
+                cb(err);
  5169
+            });
  5170
+        } else {
  5171
+            // we're not still provisioning
  5172
+            unsetTransition(payload, function (err) {
  5173
+                cb(err);
  5174
+            });
  5175
+        }
  5176
+    });
  5177
+}
  5178
+
  5179
+function svccfg(zonepath, args, callback)
  5180
+{
  5181
+    var cmd = '/usr/sbin/svccfg';
  5182
+    var options = {};
  5183
+
  5184
+    options = {
  5185
+        env: {
  5186
+            'SVCCFG_CONFIGD_PATH': '/lib/svc/bin/svc.configd',
  5187
+            'SVCCFG_REPOSITORY':
  5188
+                path.join(zonepath, 'root', '/etc/svc/repository.db')
  5189
+        }
  5190
+    };
  5191
+
  5192
+    VM.log.debug({'command': cmd + ' ' + args.join(' '), 'options': options},
  5193
+        'modifying svc repo in ' + zonepath);
  5194
+    execFile(cmd, args, options, function (error, stdout, stderr) {
  5195
+        if (error) {
  5196
+            callback(error, {'stdout': stdout, 'stderr': stderr});
  5197
+        } else {
  5198
+            callback(null, {'stdout': stdout, 'stderr': stderr});
  5199
+        }
  5200
+    });
  5201
+}
  5202
+
  5203
+
  5204
+// This calls cb() when /var/svc/provisioning is gone. When this calls cb()
  5205
+// with an Error object, the provision is considered failed so this should
  5206
+// only happen when something timed out that is unrelated to the user.
  5207
+//
  5208
+// IMPORTANT: this is only intended to be used by vmadmd. Do not use elsewhere!
  5209
+exports.waitForProvisioning = function (vmobj, cb)
  5210
+{
  5211
+    var dirname = path.join(vmobj.zonepath, 'root', '/var/svc');
  5212
+    var filename = path.join(dirname, 'provisioning');
  5213
+    var timeout;
  5214
+    var timeout_remaining = PROVISION_TIMEOUT; // default to whole thing
  5215
+    var watcher;
  5216
+
  5217
+    function done() {
  5218
+        if (timeout) {
  5219
+            VM.log.debug('clearing timeout for ' + vmobj.uuid);
  5220
+            clearTimeout(timeout);
  5221
+            timeout = null;
  5222
+        }
  5223
+        if (watcher) {
  5224
+            VM.log.debug('closing watcher for ' + vmobj.uuid);
  5225
+            watcher.close();
  5226
+            watcher = null;
  5227
+        }
  5228
+    }
  5229
+
  5230
+    if ((vmobj.state === 'provisioning')
  5231
+        && (vmobj.hasOwnProperty('transition_expire'))) {
  5232
+
  5233
+        timeout_remaining =
  5234
+            (Number(vmobj.transition_expire) - Date.now(0)) / 1000;
  5235
+
  5236
+        // Always give it at least 1 second's chance.
  5237
+        if (timeout_remaining < 1) {
  5238
+            timeout_remaining = 1;
  5239
+        }
  5240
+    } else {
  5241
+        // don't know what to do here we're not provisioning.
  5242
+        VM.log.warn('waitForProvisioning called when ' + vmobj.uuid
  5243
+            + ' was not provisioning');
  5244
+        cb();
  5245
+        return;
  5246
+    }
  5247
+
  5248
+    VM.log.debug({
  5249
+        'transition_expire': Number(vmobj.transition_expire),
  5250
+        'now': Date.now(0)
  5251
+    }, 'waiting ' + timeout_remaining + ' sec(s) for provisioning');
  5252
+
  5253
+    timeout = setTimeout(function () {
  5254
+        markVMFailure(vmobj, function (err) {
  5255
+            var errstr = 'timed out waiting for /var/svc/provisioning to move';
  5256
+            if (err) {
  5257
+                VM.log.warn(err, 'markVMFailure(): ' + err.message);
  5258
+            }
  5259
+            cb(new Error(errstr));
  5260
+            VM.log.error(errstr);
  5261
+            done();
  5262
+        });
  5263
+    }, (timeout_remaining * 1000));
  5264
+
  5265
+    VM.log.debug('created timeout for ' + vmobj.uuid);
  5266
+
  5267
+    // this starts a loop that will move provisioning -> provision_success when
  5268
+    // the hardware of the VM has been initialized the first time.
  5269
+    if (BRAND_OPTIONS[vmobj.brand].features.wait_for_hwsetup) {
  5270
+        markProvisionedWhenHWSetup(vmobj);
  5271
+    }
  5272
+
  5273
+    watcher = fs.watch(filename, function (evt, file) {
  5274
+        // We only care about 'rename' which also fires when the file is
  5275
+        // deleted.
  5276
+        VM.log.debug('watcher.event(' + vmobj.uuid + '): ' + evt);
  5277
+        if (evt === 'rename') {
  5278
+            fs.exists(filename, function (exists) {
  5279
+                if (exists) {
  5280
+                    // somehow we still have /var/svc/provisioning!
  5281
+                    markVMFailure(vmobj, function (err) {
  5282
+                        if (err) {
  5283
+                            VM.log.warn(err, 'markVMFailure(): ' + err.message);
  5284
+                        }
  5285
+                        cb(new Error('provisioning exists after rename!'));
  5286
+                        done();
  5287
+                    });
  5288
+                    return;
  5289
+                }
  5290
+
  5291
+                // So long as /var/svc/provisioning is gone, we don't care what
  5292
+                // replaced it.  Success or failure of user script doesn't
  5293
+                // matter for the state, it's provisioned now.
  5294
+                unsetTransition(vmobj, function (err) {
  5295
+                    cb();
  5296
+                    done();
  5297
+                });
  5298
+                return;
  5299
+            });
  5300
+        }
  5301
+    });
  5302
+
  5303
+    VM.log.debug('created watcher for ' + vmobj.uuid);
  5304
+
  5305
+};
  5306
+
4904 5307
 // create and install a 'joyent' or 'kvm' brand zone.
4905 5308
 function installZone(payload, callback)
4906 5309
 {
4907 5310
     var receiving = false;
  5311
+    var var_svc_provisioning = false;
4908 5312
     var vmobj;
  5313
+    var zoneinit = {};
4909 5314
 
4910 5315
     VM.log.debug('installZone()');
4911 5316
 
@@ -4961,8 +5366,9 @@ function installZone(payload, callback)
4961 5366
 
4962 5367
             zoneadm(args, function (err, fds) {
4963 5368
                 if (err) {
4964  
-                    VM.log.error('zoneadm failed to install: '
4965  
-                        + JSON.stringify(fds), err);
  5369
+                    VM.log.error({'err': err, 'stdout': fds.stdout,
  5370
+                        'stderr': fds.stderr}, 'zoneadm failed to install: '
  5371
+                        + err.message);
4966 5372
                     cb(err);
4967 5373
                 } else {
4968 5374
                     cb();
@@ -5014,8 +5420,8 @@ function installZone(payload, callback)
5014 5420
             if (!receiving) {
5015 5421
                 saveMetadata(payload, function (err) {
5016 5422
                     if (err) {
5017  
-                        VM.log.error('unable to save metadata: '
5018  
-                            + err.message, err);
  5423
+                        VM.log.error(err, 'unable to save metadata: '
  5424
+                            + err.message);
5019 5425
                         cb(err);
5020 5426
                     } else {
5021 5427
                         cb();
@@ -5030,13 +5436,156 @@ function installZone(payload, callback)
5030 5436
                 unsetTransition(vmobj, cb);
5031 5437
             } else {
5032 5438
                 cb();
  5439
+            }
  5440
+        }, function (cb) {
  5441
+            // load /var/zoneinit/zoneinit.json file if it's relevant:
  5442
+
  5443
+            var filename;
  5444
+            // var zoneinit is in installZone() scope
  5445
+
  5446
+            // when receiving zoneinit is never run.
  5447
+            if (receiving) {
  5448
+                cb();
5033 5449
                 return;
5034 5450
             }
  5451
+
  5452
+            filename = path.join(vmobj.zonepath, 'root',
  5453
+                '/var/zoneinit/zoneinit.json');
  5454
+
  5455
+            if (BRAND_OPTIONS[vmobj.brand].features.zoneinit) {
  5456
+                fs.readFile(filename, function (error, data) {
  5457
+                    if (error && (error.code === 'ENOENT')) {
  5458
+                        // doesn't exist, leave empty
  5459
+                        VM.log.debug('zoneinit.json does not exist.');
  5460
+                        cb();
  5461
+                    } else if (error) {
  5462
+                        // error reading: fail.
  5463
+                        cb(error);
  5464
+                    } else {
  5465
+                        // success try to load json
  5466
+                        try {
  5467
+                            zoneinit = JSON.parse(data.toString());
  5468
+                            VM.log.debug({'zoneinit_json': zoneinit},
  5469
+                                'parsed zoneinit.json');
  5470
+                        } catch (e) {
  5471
+                            zoneinit = {};
  5472
+                            VM.log.error(e);
  5473
+                        }
  5474
+                        cb();
  5475
+                    }
  5476
+                });
  5477
+            } else {
  5478
+                VM.log.debug('brand does not support '
  5479
+                    + '/var/zoneinit/zoneinit.json, not trying to load.');
  5480
+                cb();
  5481
+            }
  5482
+        }, function (cb) {
  5483
+            // For joyent and joyent-minimal at least, set the timeout for the
  5484
+            // svc start method to the value specified in the payload, or a
  5485
+            // default.
  5486
+
  5487
+            var timeout;
  5488
+
  5489
+            if (BRAND_OPTIONS[vmobj.brand].features.update_mdata_exec_timeout) {
  5490
+
  5491
+                if (payload.hasOwnProperty('mdata_exec_timeout')) {
  5492
+                    timeout = payload.mdata_exec_timeout;
  5493
+                } else {
  5494
+                    timeout = DEFAULT_MDATA_TIMEOUT;
  5495
+                }
  5496
+
  5497
+                svccfg(vmobj.zonepath, [
  5498
+                    '-s', 'svc:/smartdc/mdata:execute',
  5499
+                    'setprop', 'start/timeout_seconds', '=', 'count:', timeout
  5500
+                    ], function (error, stdio) {
  5501
+
  5502
+                    if (error) {
  5503
+                        VM.log.error(error, 'failed to set mdata:exec timeout');
  5504
+                        cb(error);
  5505
+                        return;
  5506
+                    }
  5507
+
  5508
+                    cb();
  5509
+                });
  5510
+            } else {
  5511
+                cb();
  5512
+            }
  5513
+
  5514
+        }, function (cb) {
  5515
+
  5516
+            var filename;
  5517
+            // var_svc_provisioning is at installZone() scope
  5518
+
  5519
+            // If we're not receiving, we're provisioning a new VM and in that
  5520
+            // case we write the /var/svc/provisioning file which should exist
  5521
+            // until something in the zone decides provisioning is complete. At
  5522
+            // that point it will be moved to either:
  5523
+            //
  5524
+            //    /var/svc/provisioning_success
  5525
+            //    /var/svc/provisioning_failure
  5526
+            //
  5527
+            // to indicate that the provisioning setup has been completed.
  5528
+
  5529
+            if (receiving) {
  5530
+                cb();
  5531
+                return;
  5532
+            }
  5533
+
  5534
+            if (BRAND_OPTIONS[vmobj.brand].features.var_svc_provisioning) {
  5535
+                // these brands always handle /var/svc/provisioning on this
  5536
+                // platform.
  5537
+
  5538
+                var_svc_provisioning = true;
  5539
+            } else {
  5540
+                if (zoneinit.hasOwnProperty('features')) {
  5541
+                    if (zoneinit.features.var_svc_provisioning) {
  5542
+                        VM.log.debug('features.var_svc_provisioning == true');
  5543
+                        var_svc_provisioning = true;
  5544
+                    }
  5545
+                } else {
  5546
+                    // didn't load zoneinit features, so check for datasets that
  5547
+                    // have 04-mdata.sh
  5548
+                    filename = path.join(vmobj.zonepath,
  5549
+                        '/root/root/zoneinit.d/04-mdata.sh');
  5550
+                    if (fs.existsSync(filename)) {
  5551
+                        VM.log.debug('/root/zoneinit.d/04-mdata.sh exists');
  5552
+                        var_svc_provisioning = true;
  5553
+                    } else {
  5554
+                        VM.log.debug('/root/zoneinit.d/04-mdata.sh does not '
  5555
+                            + 'exist');
  5556
+                    }
  5557
+                }
  5558
+            }
  5559
+
  5560
+            if (var_svc_provisioning) {
  5561
+                fs.writeFile(path.join(vmobj.zonepath, 'root',
  5562
+                    '/var/svc/provisioning'), '', function (err, result) {
  5563
+
  5564
+                    if (err) {
  5565
+                        VM.log.error(err, 'failed to create '
  5566
+                            + '/var/svc/provisioning: ' + err.message);
  5567
+                    } else {
  5568
+                        VM.log.debug('created /var/svc/provisioning in '
  5569
+                            + path.join(vmobj.zonepath, 'root'));
  5570
+                    }
  5571
+
  5572
+                    cb(err);
  5573
+                });
  5574
+            } else {
  5575
+                VM.log.debug('VM does not support /var/svc/provisioning, '
  5576
+                    + 'not creating');
  5577
+                cb();
  5578
+            }
5035 5579
         }, function (cb) {
5036 5580
             // This writes out the 'zoneconfig' file used by zoneinit to root's
5037 5581
             // home directory in the zone.
5038  
-            if (BRAND_OPTIONS[vmobj.brand].features.write_zoneconfig
5039  
-                && !receiving) {
  5582
+            if (! receiving
  5583
+                && BRAND_OPTIONS[vmobj.brand].features.zoneinit
  5584
+                && (! zoneinit.hasOwnProperty('features')
  5585
+                || zoneinit.features.zoneconfig)) {
  5586
+
  5587
+                // No 'features' means old dataset.  If we have old dataset or
  5588
+                // one that really wants a zoneconfig, write it out.
5040 5589
 
5041 5590
                 writeZoneconfig(payload, function (err) {
5042 5591
                     cb(err);
@@ -5059,7 +5608,9 @@ function installZone(payload, callback)
5059 5608
                 && BRAND_OPTIONS[vmobj.brand].features.cleanup_dataset
5060 5609
                 && !receiving) {
5061 5610
 
5062  
-                cleanupMessyDataset(vmobj.zonepath, function (err) {
  5611
+                cleanupMessyDataset(vmobj.zonepath, vmobj.brand,
  5612
+                    function (err) {
  5613
+
5063 5614
                     cb(err);
5064 5615
                 });
5065 5616
             } else {
@@ -5067,51 +5618,63 @@ function installZone(payload, callback)
5067 5618
             }
5068 5619
         }, function (cb) {
5069 5620
             // The vm is now ready to start, we'll start if autoboot is set.
5070  
-            if (payload.autoboot) {
5071  
-                VM.start(payload.uuid, {}, function (err, res) {
5072  
-                    if (err) {
5073  
-                        cb(err);
5074  
-                    } else {
5075  
-                        cb();
5076  
-                    }
5077  
-                });
5078  
-            } else {
5079  
-                cb();
5080  
-            }
5081  
-        }, function (cb) {
5082  
-            // zoneinit runs in joyent branded zones and the zone is not
5083  
-            // considered provisioned until it's rebooted once.
5084  
-            if (!BRAND_OPTIONS[vmobj.brand].features.wait_for_reboot
5085  
-                || !payload.autoboot || payload.nowait || receiving) {
5086  
-
  5621
+            if (!payload.autoboot) {
5087 5622
                 cb();
5088 5623
                 return;
5089 5624
             }
5090 5625
 
5091  
-            VM.waitForZoneState(payload, 'running', {}, function (err) {
  5626
+            VM.start(payload.uuid, {}, function (err, res) {