/
chyves.8
2791 lines (2734 loc) · 84 KB
/
chyves.8
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
.\" generated with Ronn/v0.7.3
.\" http://github.com/rtomayko/ronn/tree/0.7.3
.
.TH "CHYVES" "8" "August 27 2016" "chyves" "FreeBSD System Manager's Manual"
.
.SH "NAME"
\fBchyves\fR \- bhyve(8) front end manager \-\- version 0\.1\.0
.
.SH "SYNOPSIS"
.
.IP "\(bu" 4
\fBchyves dataset <pool> {install|upgrade}\fR
.
.IP "\(bu" 4
\fBchyves dev\fR
.
.IP "\(bu" 4
\fBchyves {firmware|iso} list\fR
.
.IP "\(bu" 4
\fBchyves {firmware|iso} import {http\-URL|ftp\-URL|local\-path}\fR
.
.IP "\(bu" 4
\fBchyves {firmware|iso} rename <resource> <new\-resource\-name>\fR
.
.IP "\(bu" 4
\fBchyves {firmware|iso} delete <resource>\fR
.
.IP "\(bu" 4
\fBchyves <guest> clone <clonenames>|MG [\-ce|\-cu|\-ie|\-iu] [<pool>]\fR
.
.IP "\(bu" 4
\fBchyves <guest> console\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all console {reset|tmux|vnc}\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG create [<size>] [<pool>]\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all delete [force|keepnet]\fR
.
.IP "\(bu" 4
\fBchyves <guest> disk add [<size>]\fR
.
.IP "\(bu" 4
\fBchyves <guest> disk disk{n} {description|notes} <annotation>\fR
.
.IP "\(bu" 4
\fBchyves <guest> disk delete disk{n}\fR
.
.IP "\(bu" 4
\fBchyves <guest> disk resize disk{n} <new\-size>\fR
.
.IP "\(bu" 4
\fBchyves <guest> disk list\fR
.
.IP "\(bu" 4
\fBchyves <guest> get {<property>|all}\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all reclaim\fR
.
.IP "\(bu" 4
\fBchyves <guest> rename <new\-guest\-name>\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|global|defaults|all set <prop1>=<value>\.\.\.\fR
.
.IP "\(bu" 4
\fBchyves <guest> snapshot [<@snapshotname>]\fR
.
.IP "\(bu" 4
\fBchyves <guest> snapshot list\fR
.
.IP "\(bu" 4
\fBchyves <guest> snapshot delete <@snapshotname>\fR
.
.IP "\(bu" 4
\fBchyves <guest> snapshot rollback [<@snapshotname>]\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all start [<iso>]\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all stop [force]\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all upgrade\fR
.
.IP "\(bu" 4
\fBchyves <guest>|MG|all unset <property>\fR
.
.IP "\(bu" 4
\fBchyves help\fR
.
.IP "\(bu" 4
\fBchyves info [\-zbprvstcdnakl|\-h]\fR
.
.IP "\(bu" 4
\fBchyves list [bridges|clones|defaults|disks|firmware|iso|pools|\e properties|<property>]\fR
.
.IP "\(bu" 4
\fBchyves list global [<pool>|primary]\fR
.
.IP "\(bu" 4
\fBchyves list {processes|snapshots} [<guest>]\fR
.
.IP "\(bu" 4
\fBchyves list tap active\fR
.
.IP "\(bu" 4
\fBchyves network <guest> add {tap|tap{n}|vale{n}}\fR
.
.IP "\(bu" 4
\fBchyves network <guest> add {tap|tap{n}} [bridge{n}]\fR
.
.IP "\(bu" 4
\fBchyves network <guest> remove {tap{n}|vale{n}}\fR
.
.IP "\(bu" 4
\fBchyves network bridge{n} attach {vlan\-iface{n}|physical\-iface{n}}\fR
.
.IP "\(bu" 4
\fBchyves network bridge{n} {default|private}\fR
.
.IP "\(bu" 4
\fBchyves network bridge{n} {join|unjoin} tap{n}\fR
.
.IP "\(bu" 4
\fBchyves network bridge{n} migrate bridge{n}\fR
.
.IP "\(bu" 4
\fBchyves upgrade [master|dev|check]\fR
.
.IP "\(bu" 4
\fBchyves version\fR
.
.IP "" 0
.
.SH "SYNTAX NOMENCLATURE:"
The following syntax nomenclature is used throughout this page:
.
.IP "\(bu" 4
\fBsubcommand\fR \- Required text, exactly as shown\.
.
.IP "\(bu" 4
\fB[subcommand]\fR \- Optional text, exactly as shown\.
.
.IP "\(bu" 4
\fB<user\-input>\fR \- User supplied input field\. Required when not contain in [ ]\.
.
.IP "\(bu" 4
\fB{n}\fR \- A whole number\. Context determines valid range\.
.
.IP "\(bu" 4
\fB\.\.\.\fR \- Repeating as many iterations as desired, follows the same preceding syntax pattern\.
.
.IP "\(bu" 4
\fB|\fR \- Separates options in a list\.
.
.IP "\(bu" 4
\fB[optional|list]\fR \- An optional field, list of valid options\. "" meaning literally no input is also an option with these lists\.
.
.IP "\(bu" 4
\fB{require|list}\fR \- A required field, list of valid options\.
.
.IP "\(bu" 4
\fB$null\fR \- \'\', meaning literally no input\.
.
.IP "\(bu" 4
\fB[\-abcdefg]\fR \- An optional flag field but must start with a \'\-\' and followed by any combination, in any order\. Eg\. \'\-gca\'
.
.IP "\(bu" 4
\fB<guest>\fR \- chyves guest name
.
.IP "\(bu" 4
\fB<pool>\fR \- ZFS pool\. The word "primary" can be used to specify the primary pool\.
.
.IP "\(bu" 4
\fB<size>\fR \- Whole number with a size suffix in kilobytes (K), megabytes (M), gigabytes (G), or terabytes (T)\.
.
.IP "\(bu" 4
\fB<resource>\fR \- ISO or UEFI firmware resource name
.
.IP "\(bu" 4
\fBMG\fR \- Multiple guests can be given using commas\. The word "all" can be used to indicate all guests for these commands\.
.
.IP "" 0
.
.SH "DESCRIPTION"
chyves(8) relies on the FreeBSD hypervisor bhyve(8), zfs(8), nmdm(4), Bourne shell sh(1), and cu(1) to start and manage type 2 virtualized guests\. Optionally grub2\-bhyve and tmux(1) can be used to expand the capabilities of chyves\. chyves uses configuration files, ZFS filesystem datasets and volumes to store guest, chyves settings, and chyves resources in an organized hierarchy\. bhyve uses virtio drivers built into the GENERIC kernel for network virtualization\. The virtio drivers are used to paravirtualize I/O for disk and network access\.
.
.P
For networking, a guest is presented a PCI Ethernet device through the use of virtio(4)\. On FreeBSD guests this means the vtnet(4) driver is used\. On 12\-CURRENT, Intel e1000 emulation was added as an alternative to virtio\-net\. In either case, the host communicates with this device through a tap(3) or vale(4) interface\.
.
.P
For storage, guests are provided block storage device(s) via ZFS volumes from the host by default\. ZFS datasets are used to organize these block devices and other resources in a hierarchical structure\. See \fIZFS\fR section for more information\.
.
.P
Multi\-guest [MG] support is the ability to specify multiple guest names that are comma separate for a sub\-command\. Not all sub\-commands have this functionality but the ones that do, allow for rapid execution of the same command over many guests, this can be incredibly helpful for a fleet of VMs\. Commands in the synopsis above with the "\fIguest\fR|MG" syntax indicate compatibility with multi\-guest\. Using the word \'all\' instead of \fIguest\fR will specify all active guests on the system for supported commands\.
.
.SH "DEPENDENCIES"
bhyve(8) and chyves(8) will run on a base FreeBSD installation\. However the required kernel modules are not loaded by default on FreeBSD\. By default \fBchyves\fR will attempt to load the required kernel modules when needed\. There are also certain non\-FreeBSD\-base binaries that enhance \fBchyves\fR capabilities\. These binaries are checked for and in certain cases, exits if the required component is not detected\.
.
.P
At least one ZFS pool is required\. All data is stored on one or many ZFS pools and depends on ZFS volumes for guest\'s disks backing\. \fBchyves\fR capitalizes on multi\-pool configurations\. Multiple pool configurations are common in FreeNAS deployments, separated storage types (eg: SSD, HD, NVMe), and OS plus storage configurations\.
.
.SS "Kernel modules:"
.
.IP "\(bu" 4
\fBvmm\fR \- Required to run the guests as this allocates the resources\.
.
.IP "\(bu" 4
\fBnmdm(4)\fR \- Required for serial console access to the guests\.
.
.IP "\(bu" 4
\fBif_tap(4)\fR \- Required for network access to the guests\.
.
.IP "\(bu" 4
\fBif_bridge(4)\fR \- Required for network access to the guests\.
.
.IP "\(bu" 4
\fBbridgestp\fR \- Required by \fBif_bridge\fR\.
.
.IP "\(bu" 4
\fBnetmap(4)\fR \- Required for VALE support\. Not compiled into GENERIC on 10\.3\.
.
.IP "" 0
.
.SS "Enhancing Binaries:"
.
.IP "\(bu" 4
\fBbhyve\fR Custom variants required for VALE or UEFI SUPPORT for pre\-11\.0\-ALPHA* builds\. Manually compile from \fB/usr/src\fR\.
.
.IP "\(bu" 4
\fBBHYVE_UEFI_20151002\.fd\fR UEFI firmware required to boot UEFI guests\. In Peter Grehan\'s public FreeBSD file directory\.
.
.IP "\(bu" 4
\fBBHYVE_UEFI_20160526\.fd\fR UEFI firmware required to boot UEFI guests with VNC support\. In Peter Grehan\'s public FreeBSD file directory\.
.
.IP "\(bu" 4
\fBgrub2\-bhyve\fR Required for booting non\-FreeBSD or non\-UEFI guests\. In ports and pkg\.
.
.IP "\(bu" 4
\fBtmux\fR Required when using the command \fBchyves <guest> console tmux\fR\. In ports and pkg\.
.
.IP "\(bu" 4
\fBvale\-ctl\fR Required for VALE networking support\. In \'/usr/src\'\.
.
.IP "" 0
.
.SS "Hardware dependencies:"
bhyve(8) is dependent on hardware virtualization CPU features\. One feature is called Second Level Address Translation (SLAT), also known as nested paging\. Intel\'s implementation is called Extended Page Tables (EPT) and AMD\'s implementation is called Rapid Virtualization Indexing (RVI)\. This CPU feature is referenced as \'POPCNT\' in dmesg(8) under \'Features2:\' line\.
.
.P
There is an early series of CPUs from Intel (Xeon 5500) that has most of the necessary virtualization capabilities but lack a feature Intel later implemented called unrestricted guest (\'\fBUG\fR\' in dmesg(8) under "VT\-x:")\. Without this feature bhyve is limited to FreeBSD guests with only one CPU core\. AMD has the \'\fBUG\fR\' functionality in all of it\'s RVI implementations\.
.
.P
PCI Pass\-through support is also provided through another sub\-set of hardware virtualization CPU features called I/O MMU virtualization\. Intel calls their implementation \'\fBVT\-d\fR\' and AMD calls their implementation \'\fBAMD\-Vi\fR\'\. See the \fBpcidev_{n}\fR property under \fIGuest Properties\fR section for instructions on attaching PCI pass\-through devices\. A check is ran before starting a guest that has PCI pass\-through devices configured and will not start the guest if the host does not have the necessary CPU feature\.
.
.SH "ZFS"
As mentioned in the \fIDEPENDENCIES\fR section above, there is a hard dependency on ZFS and as such requires at least one ZFS pool\. There are many features that rely on ZFS, such as cloning\. The ISO, UEFI firmware, and guest resources and properties are stored within ZFS datasets\.
.
.P
Single and multiple ZFS pools are supported, there is no loss in functionality for either\. \fBchyves\fR datasets get mounted to \'/chyves/\fIpool\fR\'\. The \'/chyves\' directory itself is stored on \'/\' and does not belong to any of the chyves datasets\. The first pool is assigned the primary pool role but this can be changed afterwards\. The primary pool is the only pool containing the UEFI Firmware resources, ISO resources, the global properties, and the comprehensive logs\. The primary pool can be changed with \fBchyves dataset <new\-prim\-pool> promote\fR which moves the resources and global configuration to \fB<new\-prim\-pool>\fR from the current primary pool\.
.
.P
See \fIDATASET SUB\-COMMANDS\fR section below for managing the \fBchyves\fR datasets\.
.
.P
Example of a multi\-pool folder structure:
.
.IP "" 4
.
.nf
/chyves/prim_pool/guests/
/guests/<guest1>/\.config/\.cfg # File
/guests/<guest1>/disk{n} # Volumes
/guests/<guest1>/img # Dataset(s)
/guests/<guest1>/log # Dataset
/guests/<guest1>/log/YYYYMM\.log # Files
/guests/<guest2>/\.config/\.cfg # File
/guests/<guest2>/disk{n} # Datasets
/guests/<guest2>/img # Dataset(s)
/guests/<guest2>/log # Dataset
/guests/<guest2>/log/YYYYMM\.log # Files
/guests/\.defaults # File
/\.config # Dataset
/\.config/global\.cfg # File
/\.config/pool\.cfg # File
pool_role=primary # Pool property
/logs # Dataset
/logs/YYYYMM\.log # Files
/ISO # Datasets
/Firmware # Datasets
/chyves/secd_pool/guests
/guests/<guest3>/\.config/\.cfg # File
/guests/<guest3>/disk{n} # Volumes
/guests/<guest3>/img # Dataset(s)
/guests/<guest3>/log # Dataset
/guests/<guest3>/log/YYYYMM\.log # Files
/\.config # Dataset
/\.config/pool\.cfg # File
pool_role=secondary # Pool property
/chyves/anther_pl/guests
/guests/<guest4>/\.config/\.cfg # File
/guests/<guest4>/disk{n} # Volumes
/guests/<guest4>/img # Dataset(s)
/guests/<guest4>/log # Dataset
/guests/<guest4>/log/YYYYMM\.log # Files
/\.config # Dataset
/\.config/pool\.cfg # File
pool_role=secondary # Pool property
.
.fi
.
.IP "" 0
.
.P
Offline dataset:
.
.IP "" 4
.
.nf
offlinePL/chyves_offline # Datasets
offlinePL/chyves_offline/\.config/pool\.cfg # File
pool_role=offline # Pool property
.
.fi
.
.IP "" 0
.
.P
Example of a single pool dataset hierarchy:
.
.IP "" 4
.
.nf
/chyves/zroot/guests
/guests/<guest1>/\.config/\.cfg # File
/guests/<guest1>/disk{n} # Volumes
/guests/<guest1>/img # Dataset(s)
/guests/<guest1>/log # Dataset
/guests/\.defaults # File
/\.config # Dataset
/\.config/global\.cfg # File
/\.config/pool\.cfg # File
pool_role=primary # Pool property
/logs # Dataset
/ISO # Datasets
/Firmware # Datasets
.
.fi
.
.IP "" 0
.
.SS "GUEST STORAGE"
By default, chyves uses ZFS volumes as guest storage backing\. Each volume is named disk{n} and is attached to the guest via an emulated AHCI controller\.
.
.P
There is limited support for raw file backed guest storage\. This is a manual process and there are not any chyves commands that create, modify, or delete these raw image files\. \fBAll files\fR in the \'img/\' folder and child folders are attached to the guest on start\. To have independent snapshots for each raw image, create a child datasets under \fB<pool>/chyves/guests/<guest>/img\fR, this has an advantage over ZFS volumes as there is no refreservation cost associated with the snapshots\. The command \fBchyves info\fR reports the raw image files and size\. To create a sparse raw image run:
.
.P
\fBtruncate \-s 16G /chyves/<pool>/guests/<guest>/img/<image\-name>\fR\.
.
.SH "LOGS"
All logs entries are stored on the primary pool under \fB/chyves/<primary\-pool>/chyves/logs\fR with the dataset\'s compression turned on\. The file are separated on a monthly basis using a \fBYYYYMM\.log\fR file name\. The log entries use the ISO 8601 date format for monitoring tools\. There is a separate log file on a per guest basis, however at this time not all guest events are recorded to this file, however these events \fIare\fR recorded in the global logs regardless\.
.
.IP "" 4
.
.nf
2016\-07\-09T21:27:00+0000 \- [2] \- Mimicking tap50 interface for g10\.
.
.fi
.
.IP "" 0
.
.P
The logs are dash delimited with leading and trailing single spaces\. The first field indicates the time in ISO 8601 format, \'2016\-07\-09T21:27:00+0000\' in the example above\. The second field is the \'stdout_level\', \'2\' in the example above\. The third field is the log event, \'Mimicking tap50 interface for g10\.\' in the example above\. See the \fIGlobal Configuration Properties\fR section below for more information on what \'stdout_level\' does\.
.
.SH "NETWORK"
The network mode is set by the global setting \'\fBnetwork_design_mode\fR\' and determines how chyves handles network devices for the guests\. The two modes are \'\fBauto\fR\' and \'\fBsystem\fR\', both modes are discussed below\.
.
.P
VALE support is experimentally provided as a proof of concept\. VALE support is not widely implemented and is feature limited at it\'s current development stage\. To use VALE requires that \'device netmap\' be compiled into the kernel and that \'vale\-ctl\' be compiled from \'/usr/src/tools/tools/netmap\' and moved to $PATH directory\. VALE natively support em(4), igb(4), ixgbe(4), and re(4)\. Other NICs must use emulation which may reduce performance\. See netmap(4) for more information\. VALE interfaces are added to a guest the same way a tap interface is added using: \'\fBchyves network add vale{n}[:{p}]\fR\' where {n} is the vale switch you want to use and optionally :{p} specifies the port on that switch\. Specifying a port is not recommended\. VALE interfaces are clever in that the interface is commonly shared between other guests connecting to that \'virtual\' VALE switch\. Two guests using \'vale0\' will be connected to the same virtual switch\.
.
.P
In 12\-CURRENT r302504, guest Intel e1000 emulation was added as an alternative to using virtio\-net\. This can be utilized by setting \'bhyve_net_type\' to \'e1000\' from a 12\-CURRENT host on a per guest basis\.
.
.SS "AUTO DESIGN NETWORK MODE"
By default, chyves is configured to manage the complete network design layout\. chyves keeps track of which physical, tap, and vlan interfaces are associated with which bridges, and which tap and VALE interfaces are associated with which guests\. Upon starting a guest these designs are checked, verified, and then created if not existing\. To manage the network design using this method, use the \'chyves network\' commands to create the associations and \'chyves list bridges\' for an overview\.
.
.P
On a fresh simple installation, the only command needed is \'chyves network bridge0 attach em0\'\. This tells chyves to associate the interface \'em0\' with the default bridge\. To change the default bridge, the command would be \'chyves network bridge64 default\' to set \'bridge64\' as the default\. All tap interfaces created for guests are then associated with the bridge set in defaults\.
.
.P
As indicated above, tap interfaces are attached to a bridge, which optionally can be attached to a physical or vlan interface\. If the bridge is not associated with a physical or vlan interface, the bridge must be put into \'private\' mode\. This is done by running \'chyves network bridge64 private\'\. A private bridge allows sensitive traffic to transverse the bridge without the possibility of traffic being monitored from an outside network\. A private bridge can also use pf(4) for NAT capabilities but this is not built into chyves\.
.
.SS "SYSTEM DESIGN NETWORK MODE"
The \'system\' mode assumes the user will correctly create and configure all devices on the host system, including the tap devices\. No checks are ran to test connectivity and chyves simply attaches tap or VALE interfaces to the guest as configured in the \'net_ifaces\' property\.
.
.P
Converting from \'system\' to \'auto\' will not automatically start to work\. Each device will need to be manually associated with each other\.
.
.SH "PROPERTIES"
There are four types of chyves properties: global configuration properties, pool properties, per\-guest properties, and default guest properties\. Each is stored in a file in the chyves dataset hierarchy\. \fBIt is not recommended to directly edit the \'*\.cfg\' files, this is because there are no filters to stop typos, warn for incorrect property names, or invalid values\.\fR The filter is only applied when running the \'chyves \fIguest\fR set\' command and not rechecked when pulling the values from the files\.
.
.P
Also certain actions are triggered when setting certain properties\. For example, setting the \'\fBtemplate\fR\' guest property will set or unset the \fIreadonly\fR flag on the ZFS dataset\.
.
.P
The pool configuration is stored in a file at \'/chyves/\fIpool\fR/\.config/pool\.cfg\'\. On the primary pool there is also a file in the same directory called \'global\.cfg\', this contains the global settings for chyves\.
.
.P
The \'dataset_role\' setting is used to store the role of that pool containing the chyves datasets\. Valid values are primary, secondary, and offline\. Only one \'primary\' dataset can be configured per system\. The primary dataset is the only dataset that contains the ISO and firmware resources\. Only the primary pool has guest defaults configuration file\. These defaults are referenced when setting properties for new guests\. The primary pool is also where new guests are created when the [\fIpool\fR] argument is not declared with \'chyves create\' command\. All other active pools are set to the "secondary" role\. Pools set to the \'offline\' role are left untouched and is a good role to put a local replicated backup\. The primary pool is critical to the operation of chyves, without a primary pool set, only the \'dataset\' sub\-command are permitted to run\. The first pool setup on the system is the primary pool\. This can be changed later with \'chyves dataset \fIpool\fR promote\'\.
.
.SS "Pool Properties"
These properties keeps track of the version of the dataset and the role\. With multiple pools, defined per\-pool roles are necessary\. This is defined through the use of a dataset called \'\fIpool\fR/chyves/\.config\' where the pool specific properties (\fBpool\.cfg\fR) are stored\.
.
.IP "\(bu" 4
\fBdataset_role={primary|secondary|offline}\fR
.
.IP
\fBManaged through \'chyves dataset\' commands\.\fR
.
.IP
Vital in multi\-pool configurations, each role type has different characteristics as explained below:
.
.IP
\fBprimary\fR \- Pools in this role are utilized exclusively for the following purposes: Store ISO and Firmware resources\. Reference point for global properties\. New guest are defaulted to be created on this pool unless otherwise specified as a property for \'chyves create\'\. Only one ZFS pool can be in the primary role per system and is the only role for single zpool configurations\.
.
.IP
\fBsecondary\fR \- Pools in this role are utilized for storing other active guests not stored on the primary ZFS pool\.
.
.IP
\fBoffline\fR \- Pools in this role are not utilized, this role is intended for backup\. Guests on these pools are omitted from all subcommands\. To further prevent action on \'offline\' pools, the dataset is renamed to \'chyves_offline\' which chyves will completely ignore\.
.
.IP "\(bu" 4
\fBpool_version={dataset\-version}\fR
.
.IP
\fBManaged through \'chyves dataset\' commands\.\fR
.
.IP
This property indicates the current version of the chyves pool\. This an upgrade mechanism to ensure the dataset and it\'s child datasets contain the correct architecture\. The command \'chyves dataset \fIpool\fR upgrade\' brings a dataset incrementally to the latest version when there is a discrepancy between the dataset version and the chyves version\. A check is ran before execution to ensure the version of the pool is compatible with the version of chyves\.
.
.IP "" 0
.
.SS "Global Configuration Properties"
These properties control the behavior of chyves\.
.
.P
Many of these properties are not recommended to be changed directly unless the scope of their impact is understood or when managed through subcommands such as \'\fBchyves network\fR\'\. The preferred management method is given on the first line after each property in bold text\.
.
.P
Usage:
.
.P
\fBchyves global set <property>=<value>\fR
.
.P
\fBchyves list global [<pool>]\fR
.
.P
Properties:
.
.IP "\(bu" 4
\fBauto_load_kernel_mods\fR={yes|no}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
When set to "yes", chyves will load the kernel modules necessary for guests\. When set to \'no\', chyves will simply check if the modules are loaded and exit with a message if not loaded\.
.
.IP "\(bu" 4
\fBbridge{n}_phy_attach\fR={\fIinterface\fR|private}
.
.IP
\fBManaged through \'chyves network\' commands\.\fR
.
.IP
This property contains a physical or vlan interface name for a bridge to belong\. When set to \'private\' the bridge is not joined with a outside network\.
.
.IP
Example values:
.
.IP
\fBbridge0_attach=em0\fR
.
.IP
\fBbridge10_phy_attach=vlan52\fR
.
.IP
\fBbridge512_phy_attach=private\fR
.
.IP
Usage:
.
.IP
\fBchyves network bridge{n} attach {<interface>|private}\fR
.
.IP "\(bu" 4
\fBbridge{n}_tap_members\fR={tap{n}[,tap{n},tap{n}]}
.
.IP
\fBManaged through \'chyves network\' commands\.\fR
.
.IP
This property is a comma separated list of the tap members for bridge{n}, where {n} is the interface number\. Valid values for {n} are 0 to 32768\. Guests are automatically added to this property using the bridge{n} that is set in \'defaults\' under the \'bridge\' property\.
.
.IP
Examples values:
.
.IP
\fBbridge0_tap_members=t0\fR
.
.IP
\fBbridge70_tap_members=t20,t30\fR
.
.IP
A tap can be moved to another bridge by first removing the tap from the original bridge and adding to the desired bridge\. Use the following command syntax to do this:
.
.IP
\fBchyves network bridge{n} unjoin tap{n}\fR
.
.IP
\fBchyves network bridge{n} join tap{n}\fR
.
.IP "\(bu" 4
\fBcheck_for_updates\fR={daily|weekly|monthly|always}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
This determines how often to check for updates for chyves on the GitHub repository\. The keyword \'always\' means to check every time chyves is ran\.
.
.IP "\(bu" 4
\fBcheck_for_updates_last_check\fR=YYYYMMDD
.
.IP
\fBManaged internally\.\fR
.
.IP
This contain the date of last check\.
.
.IP "\(bu" 4
\fBcheck_for_updates_last_check_status\fR={1|0}
.
.IP
\fBManaged internally\.\fR
.
.IP
When set to \'1\' an update is available\.
.
.IP "\(bu" 4
\fBcheck_for_updates_timeout_seconds\fR={n}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
This number is used to set the network timeout for fetch(1) before giving up\.
.
.IP "\(bu" 4
\fBcheck_for_updates_unique_id\fR=UUID
.
.IP
\fBManaged internally\.\fR
.
.IP
This contains a UUID used for identification\. It is set when the primary pool was created\.
.
.IP "\(bu" 4
\fBconsole_start_offset\fR={integer}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
This is a compatibility mechanism used to offset the first chyves null console modem number so that a collision are less likely with another application or bhyve front end manager\. The initial default is set to \'50\'\. After the first guests are created, then the next highest console number is used to create the next guest\.
.
.IP "\(bu" 4
\fBconsolidate_bhyve_pci_devices\fR={no|yes}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
The default is set to \'no\' as it may have performance implications, however this may be necessary if more than 26 PCI devices are connect to a guest\. This property signals the bhyve string generators to use all PCI functions for similar devices\. In ideal conditions up to 208 PCI devices can be attached when set to \'yes\' on 11\.0\-RELEASE and earlier\. On 12\-CURRENT or later up to 832 device could be connected in ideal conditions\.
.
.IP "\(bu" 4
\fBdefault_clone_flag\fR
.
.IP
\fBUser managed and settable\.\fR
.
.IP
Flag to use when cloning a guest and the clone type flag is omitted\. See clone command under \fIGUEST CLONE SUB\-SUB\-COMMAND\fR section\.
.
.IP "\(bu" 4
\fBdefault_info_flags\fR
.
.IP
\fBUser managed and settable\.\fR
.
.IP
Flags set here are passed to \'chyves info\' when no flags are used\. See command \'chyves info\' for valid flags\.
.
.IP "\(bu" 4
\fBdefault_list_flags\fR
.
.IP
\fBUser managed and settable\.\fR
.
.IP
Flags set here are passed to \'chyves info\' WHEN the command \'chyves list\' is used\. Both commands use the same backend function and \'chyves list\' is kept around for legacy reasons\. See command \'chyves info\' for valid flags\.
.
.IP "\(bu" 4
\fBdev_mode\fR={off|on|\-xvn}
.
.IP
\fBUser managed and settable, advanced feature\.\fR
.
.IP
Setting to \'on\' allows for functions to be called direct from the command line using \fBchyves dev\fR\. Using the \fB\-xvn\fR flags instead of the word \fBon\fR will use Bourne\'s \fBset\fR command to turn on special option flags during \fBchyves dev\fR execution\. These flags can be used individually (\-x|\-v|\-n) or combined (\-xv)\. See sh(1)\.
.
.IP "\(bu" 4
\fBeject_iso_on_n_reboot\fR={n}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
See same property name under the \fIGuest Properties\fR section below for description of functionality\.
.
.IP "\(bu" 4
\fBlog_mode\fR={host|guest|dual}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
Determines the source of information to save in the log files when \'log_to_file\' is set to \'yes\'\. When set to \'guest\' only guest actions are recorded\. When set to \'host\' only actions not involving guests are recorded\. When set to dual, both are recorded\.
.
.IP "\(bu" 4
\fBlog_to_file\fR={yes|no}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
Turns on logging to a file\. For general logging the file is saved at \'/chyves/logs/YYYYMM\.log\' and for guests the file is saved at \'/chyves/guests/\fIguest\fR/log/YYYYMM\.log\' or at \'/chyves/\fIpool\fRguests/\fIguest\fR/ log/YYYYMM\.log\'\.
.
.IP "\(bu" 4
\fBnetwork_design_mode\fR={auto|system}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
The default is \'auto\', which means chyves manages everything above the physical or vlan Ethernet interface, all the way to the guest\. This is managed through \'chyves network\' commands\. When set to \'system\', chyves only attaches the tap or VALE interfaces to the guest and leaves it to the user to have configured everything else (correctly)\. See \fINETWORK\fR section above\.
.
.IP "\(bu" 4
\fBrestrict_new_property_names\fR=[off|on]
.
.IP
\fBUser managed and settable, advanced feature\.\fR
.
.IP
Setting to "on" allows for new properties to be created with \'chyves set\'\. The default is set to "off" to prevent creating new properties due to typos\. Even when set to \'on\' this will not allow a creation of new global or defaults properties\.
.
.IP "\(bu" 4
\fBstdout_level\fR={0\-3}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
Determines the amount of output to send to the terminal\. The four numeric settings do the following\. This does not impact the log files\.
.
.IP
\fB0 \- off\fR \- No output
.
.IP
\fB1 \- minimal\fR Sub command action shown
.
.IP
\fB2 \- regular\fR + Each step in the sub command shown
.
.IP
\fB3 \- verbose\fR ++ Outputs bhyve and loader command used\.
.
.IP
\fB4\fR \-\- Never print to screen\. Used in \fB__return*\fR functions that would break even with the value set to \'0\'\.
.
.IP "\(bu" 4
\fBtap_start_offset\fR={integer}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
This is a compatibility mechanism used to offset the first chyves tap interface so that a collision is less likely with another application or bhyve front end manager\. The initial default is set to "50", valid values are from 0 to 32767\.
.
.IP "\(bu" 4
\fBtap_up_by_default\fR={off|on}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
When set to "yes", the sysctl: net\.link\.tap\.up_on_open is set to "1" when running script\. Default is "on" but it is recommended to set to "off" and update your /boot/loader\.conf to set this sysctl\.
.
.IP "\(bu" 4
\fBvlan_iface_base_name\fR=\fIvlan_base_name\fR
.
.IP
\fBUser managed and settable\.\fR
.
.IP
When using a non\-standard vlan naming nomenclature this needs to reflect the change\. The default is "vlan" when using the standard "vlan{n}" naming nomenclature\.
.
.IP "\(bu" 4
\fBuefi_vnc_port_start_offset\fR={integer}
.
.IP
\fBUser managed and settable\.\fR
.
.IP
This is a compatibility mechanism used to offset the first VNC port used by UEFI guests\. The initial default is set to "5900", valid values are from 1 to 65536\.
.
.IP "" 0
.
.SS "Guest Properties"
These are properties used by chyves to specify the properties to pass to bhyve, grub\-bhyve, and chyves itself when starting a guest\. Changing these values on a running guest will not yield any change until the guest is completely stop and then started again\.
.
.P
Usage:
.
.P
\fBchyves <guest> get <property>\fR
.
.P
\fBchyves <guest> get all\fR
.
.P
\fBchyves list properties\fR
.
.P
\fBchyves list <property>\fR
.
.P
\fBchyves <guest>|MG|all set <property1>=<value> [<property2>=<value>] [<property3>=<value>]\fR
.
.P
\fBchyves <guest>|MG|all set <property1>=<value> [<property2>=<value>] [<property3>=<value>] [<guest2> [<property1>=<value>] [<property2>=<value>]]\fR
.
.IP "\(bu" 4
\fBbargs\fR
.
.IP
For advanced users to pass additional flags directly to \fBbhyve\fR\. See bhyve(8) for complete list of flag options\. To find out which flags are used when starting a specific guest, turn on global configuration \'stdout_level\' to \'3\' or check the global log file\.
.
.IP "\(bu" 4
\fBbhyve_net_type\fR={virtio\-net|e1000}
.
.IP
Sets the method to attach tap interfaces to the guest\. Intel e1000 emulation is available on hosts on 12\-CURRENT after commit r302504, 11\-STABLE after commit r304424, and on 10\-STABLE after commit r304425\.
.
.IP "\(bu" 4
\fBcpu\fR={n}
.
.IP
Assigns {n} number of CPU cores to a guest, up to 16\.
.
.IP "\(bu" 4
\fBchyves_guest_version\fR=\fIMMmm\fR
.
.IP
\fBManaged through \'chyves \fIguest\fR upgrade\' command\.\fR
.
.IP
Used to ensure the guest contains latest properties when moved between hosts on different versions of chyves\. "MM" signifies the major version as an integer and "mm" as the minor version as an integer\. Changes to the major version require an upgrade via \'chyves \fIguest\fR upgrade\' because a guest property value format has changed or a new required property was added\. Differences in the minor version do no require an upgrade but may not have full functionality\.
.
.IP "\(bu" 4
\fBdescription\fR="\fIINSERT BRIEF DESCRIPTION\fR"
.
.IP
Used to describe guest\. Use double quotes when description contains spaces\.
.
.IP "\(bu" 4
\fBeject_iso_on_n_reboot\fR={n}
.
.IP
This property determines if and when to eject the optical media when installing an OS\. By default this is set to \'1\' because most *nix systems install on the first boot\. Live CD should be turned to \'0\' and Windows systems should be set to \'2\' or greater\.
.
.IP
There is a global property which shares the same name has has the same functionality\. The global property is used if not set on the guest\.
.
.IP "\(bu" 4
\fBloader\fR={bhyveload|uefi|grub\-bhyve}
.
.IP
Tells which loader to use to boot guest\. \fBbhyveload\fR is for FreeBSD based guests, \fBuefi\fR is used for UEFI based guests, and \fBgrub\-bhyve\fR is used for all other guests\.
.
.IP
When setting a guest to use the \fBuefi\fR loader, some \fBuefi_*\fR properties are populated\. One of these properties is \fBuefi_firmware\fR which is required for guests to start\.
.
.IP "\(bu" 4
\fBnet_ifaces\fR=tap{n}|vale{n}[:{p}]
.
.IP
\fBManaged through \'chyves network\' commands\.\fR
.
.IP
Stores which tap and VALE interfaces are assigned to a guest\. Where {n} is a unique interface number\. Taps are process locked and can not be shared simultaneously by multiple guests\. Multiple devices are stored using commas as delimiters\.
.
.IP
VALE interface names must be vale{n}[:{p}] where {n} can be alpha\-numeric up to 9 characters long\. Optionally a port can be specified where :{p} starts with a separating colon and either a single alphabetical character or a number 0\-9999\.
.
.IP
Usage:
.
.IP
\fBchyves network <guest> add {tap|tap{n}|vale{s}[:{p}]}\fR
.
.IP
\fBchyves network <guest> add {tap|tap{n}} bridge{n}\fR
.
.IP
\fBchyves network <guest> remove {tap{n}|vale{s}[:{p}]}\fR
.
.IP
See \fINETWORK SUB\-COMMANDS\fR section for complete list of commands to manage this property\.
.
.IP "\(bu" 4
\fBnotes\fR="\fIINSERT FURTHER NOTES\fR"
.
.IP
Used to further describe guests\. Use double quotes when description contains spaces\.
.
.IP "\(bu" 4
\fBos\fR=\fIoperating\-system\fR
.
.IP
Used to determine the properties to use for the \fBgrub\.cfg\fR and \fBdevice\.map\fR files when the \fBloader\fR property is set to \fBgrub\-bhyve\fR\.
.
.IP
Supported values are openbsd59, openbsd58, openbsd57, netbsd, debian, d8lvm, centos6, centos7, arch, gentoo, default, or custom\. Any other values are invalid\. \fBdefault\fR is the generic catch\-all value\.
.
.IP
Using \fBcustom\fR relies on the user to manually create the \fBgrub\.cfg\fR and \fBdevice\.map\fR files in the guest directory \fB/chyves/<pool>/guests/<guest>\fR\.
.
.IP
Example:
.
.IP
For an OpenBSD guest "obsd59" located in \fB/chyves/kubota/obsd59/\fR and an install ISO in \fB/chyves/dozer/ISO/install59\.iso/\fR, your files will look like this:
.
.IP
\fBdevice\.map\fR file:
.
.IP "" 4
.
.nf
(hd0) /dev/zvol/kubota/chyves/obsd59/disk0
(cd0) /chyves/dozer/ISO/install59\.iso/install59\.iso
.
.fi
.
.IP "" 0
.
.IP
\fBgrub\.cfg\fR file for installation:
.
.IP "" 4
.
.nf
kopenbsd \-h com0 (cd0)/5\.9/amd64/bsd\.rd
boot
.
.fi
.
.IP "" 0
.
.IP
\fBgrub\.cfg\fR file after installation is complete:
.
.IP "" 4
.
.nf
kopenbsd \-h com0 \-r sd0a (hd0,openbsd1)/bsd
boot
.
.fi
.
.IP "" 0
.
.IP
The \fBchyves\fR project welcomes any OS submissions to help other users from re\-discovering leg work done by others\.
.
.IP "\(bu" 4
\fBpcidev_{n}\fR={bhyve\-pci\-dev}
.
.IP
This is an advance feature and assumes you have read bhyve(8) AND understood it\. This property does NOT have a complete input verification suite or complete host support verification\.
.
.IP
This property attaches a bhyve PCI device to the guest\. The \'{n}\' is completely arbitrary number and is only a matter of the order of attachment\. \'{bhyve\-pci\-dev}\' is the bhyve PCI device to attach\. See bhyve(8) for a complete list or below for common uses\.
.
.IP
\fBchyves\fR attaches each custom PCI device alone on the PCI bus, except for pass\-through devices\.
.
.IP
To create a virtio RNG, the command would be
.
.IP
\fBchyves <guest> set pcidev_0=virtio\-rnd\fR
.
.IP
To create a permanently attached optical media device using a chyves ISO resource where "cd\.iso" is the ISO name, the command would be:
.
.IP
\fBchyves <guest> set pcidev_1=ahci\-cd,/chyves/<primary\-pool>/ISO/cd\.iso/cd\.iso\fR
.
.IP
To create a sparse 16G disk and attach, the commands would be:
.
.IP
\fBtruncate \-s 16G /chyves/<pool>/guests/<guest>/01\.img\fR
.
.IP
\fBchyves <guest> set pcidev_2=ahci\-hd,/chyves/<pool>/guests/<guest>/01\.img[,block\-options]\fR
.
.IP
Although not recommended, a tap interface can be manually attached using this command:
.
.IP
\fBchyves <guest> set pcidev_3=virtio\-net,tap10\fR
.
.IP
\&\.\.\.using Intel e1000 emulation (only 12\-CURRENT hosts):
.
.IP
\fBchyves <guest> set pcidev_5=e1000,tap10\fR
.
.IP
\&\.\.\.with an associated mac address:
.
.IP
\fBchyves <guest> set pcidev_4=virtio\-net,tap10,mac=58:9C:FC:00:00:00\fR
.
.IP
PCI pass\-through devices which share the same physical PCI slot number on the host also will share the same virtual physical slot number in the guest\. This is due to the potential for kernel panics on certain devices such as Fiber Channel PCI cards\. Keep in mind, only eight PCI functions can be assigned to a virtual physical slot in \fBbhyve\fR, additional devices are ignored\. To set a PCI pass\-through device use the follow commands, where \'6\' and \'7\' are arbitrary values and \'9/0/0\' and \'9/0/1\' is the PCI bus/slot/function device triplet\.
.
.IP
\fBchyves <guest> set pcidev_6=passthru,9/0/0\fR
.
.IP
\fBchyves <guest> set pcidev_7=passthru,9/0/1\fR
.
.IP
To get the PCI triplet on the host, run:
.
.IP "" 4
.
.nf
root@bhost:~ # pciconf \-lv | grep \-A4 \-E \'^ppt\'
ppt0@pci0:9:0:0: class=0x0c0400 card=0x01461077 chip=0x24321077 rev=0x02 hdr=0x00
vendor = \'QLogic Corp\.\'
device = \'ISP2432\-based 4Gb Fibre Channel to PCI Express HBA\'
class = serial bus
subclass = Fibre Channel
ppt1@pci0:9:0:1: class=0x0c0400 card=0x01461077 chip=0x24321077 rev=0x02 hdr=0x00
vendor = \'QLogic Corp\.\'
device = \'ISP2432\-based 4Gb Fibre Channel to PCI Express HBA\'
class = serial bus
subclass = Fibre Channel
.
.fi
.
.IP "" 0
.
.IP
Normally the device in the example above would use the \fBisp(4)\fR drivers\. However, the devices need to be using the \fBppt\fR drivers\. In this context, the \fBppt\fR drivers tell the host to ignore the device as it will be pass\-through to a virtualized guest\. To get a PCI device to load the \fBppt\fR drivers instead of it\'s normal drivers, edit the \fB/boot/loader\.conf\fR file to contain the PCI triplet of each device in the \fBpptdevs\fR directive\. The following is consistent with our examples above:
.
.IP "" 4
.