From dfbed3860b7b320e1ce478baec59d543d0cddd95 Mon Sep 17 00:00:00 2001 From: Yggdrasil-Bot Date: Sun, 10 Oct 2021 12:15:40 -0400 Subject: [PATCH] REL: 1.0.2 release (Intel GVT-g + Looking Glass support) --- .gitignore | 3 + CHANGLOG.md | 10 +- README.md | 38 +- example/amd-mdev-preinstall.yaml | 37 ++ example/amd-mdev.yaml | 24 +- example/intel-mdev-preinstall.yaml | 37 ++ example/intel-mdev.yaml | 37 ++ example/nvidia-mdev-preinstall.yaml | 37 ++ example/nvidia-mdev.yaml | 37 +- libvfio.nimble | 2 +- optional/.gitkeep | 0 patches/twelve.patch | 264 ++++++++++++ scripts/install.sh | 120 ++++++ src/arcd.nim | 11 +- src/{libvfiopkg => libvfio}/comms.nim | 0 src/{libvfiopkg => libvfio}/comms/ls.nim | 0 src/{libvfiopkg => libvfio}/comms/ps.nim | 0 src/{libvfiopkg => libvfio}/comms/qmp.nim | 0 src/{libvfiopkg => libvfio}/control.nim | 2 + src/libvfio/control/arguments.nim | 381 ++++++++++++++++++ src/libvfio/control/introspection.nim | 96 +++++ src/{libvfiopkg => libvfio}/control/iommu.nim | 59 ++- src/{libvfiopkg => libvfio}/control/vm.nim | 54 ++- src/{libvfiopkg => libvfio}/logger.nim | 0 src/{libvfiopkg => libvfio}/types.nim | 0 src/{libvfiopkg => libvfio}/types/config.nim | 80 +++- .../types/connectivity.nim | 0 .../types/environment.nim | 0 .../types/hardware.nim | 4 + src/{libvfiopkg => libvfio}/types/locks.nim | 1 + src/{libvfiopkg => libvfio}/types/process.nim | 0 src/{libvfiopkg => libvfio}/types/qmp.nim | 0 .../utils/findPorts.nim | 0 .../utils/getLocks.nim | 0 .../utils/separateVfios.nim | 0 src/libvfiopkg/control/arguments.nim | 221 ---------- 36 files changed, 1244 insertions(+), 311 deletions(-) create mode 100644 example/amd-mdev-preinstall.yaml create mode 100644 example/intel-mdev-preinstall.yaml create mode 100644 example/intel-mdev.yaml create mode 100644 example/nvidia-mdev-preinstall.yaml create mode 100644 optional/.gitkeep create mode 100644 patches/twelve.patch create mode 100755 scripts/install.sh rename src/{libvfiopkg => libvfio}/comms.nim (100%) rename src/{libvfiopkg => libvfio}/comms/ls.nim (100%) rename src/{libvfiopkg => libvfio}/comms/ps.nim (100%) rename src/{libvfiopkg => libvfio}/comms/qmp.nim (100%) rename src/{libvfiopkg => libvfio}/control.nim (69%) create mode 100644 src/libvfio/control/arguments.nim create mode 100644 src/libvfio/control/introspection.nim rename src/{libvfiopkg => libvfio}/control/iommu.nim (83%) rename src/{libvfiopkg => libvfio}/control/vm.nim (77%) rename src/{libvfiopkg => libvfio}/logger.nim (100%) rename src/{libvfiopkg => libvfio}/types.nim (100%) rename src/{libvfiopkg => libvfio}/types/config.nim (76%) rename src/{libvfiopkg => libvfio}/types/connectivity.nim (100%) rename src/{libvfiopkg => libvfio}/types/environment.nim (100%) rename src/{libvfiopkg => libvfio}/types/hardware.nim (96%) rename src/{libvfiopkg => libvfio}/types/locks.nim (95%) rename src/{libvfiopkg => libvfio}/types/process.nim (100%) rename src/{libvfiopkg => libvfio}/types/qmp.nim (100%) rename src/{libvfiopkg => libvfio}/utils/findPorts.nim (100%) rename src/{libvfiopkg => libvfio}/utils/getLocks.nim (100%) rename src/{libvfiopkg => libvfio}/utils/separateVfios.nim (100%) delete mode 100644 src/libvfiopkg/control/arguments.nim diff --git a/.gitignore b/.gitignore index 3ab2bdc..52d8c37 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ * !*.* !*/ +!*/.gitkeep *.exe @@ -17,3 +18,5 @@ $RECYCLE.BIN/ # Ignoring binary files produced by debugger. nimcache/ + +optional/ diff --git a/CHANGLOG.md b/CHANGLOG.md index 02165b0..809ae81 100644 --- a/CHANGLOG.md +++ b/CHANGLOG.md @@ -1,6 +1,14 @@ +# 1.0.2 + +- Intel GVT-g support +- Looking Glass support +- Inter-VM Shared Memory (IVMSHMEM) resource management +- Automated mediated device provisioning +- Installer script (Supports Ubuntu 20.04) + # 1.0.1 -- NVidia Support (nvidia-mdev only currently) +- Nvidia Support (nvidia-mdev only currently) - Additional Documentations - Deployment script - Undeployment script diff --git a/README.md b/README.md index 1ebe137..e710d46 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ -# LibVFIO +# LibVF.IO -This is a project designed to serve as a replacement to -libvirt, with a focus on GPU/Compute device multitenancy. +LibVF.IO is a vendor neutral GPU multiplexing tool driven by YAML & VFIO. # Documentation -Here are some key components to the documentation that we -currently have: +The following pages provide a basic introduction to LibVF.IO: 1. [Deploy](docs/deployment.md) 2. [Using](docs/using.md) @@ -15,23 +13,25 @@ currently have: # Release Features -1. AMDGPU mediated device support -2. Nvidia mediated device support -3. YAML Configuration files -4. Create VM -5. Start VM -6. Stop VM -7. List available kernels -8. List available states -9. List running kernels -10. Deploy script -11. Undeploy script +1. Automated installation +2. Intel mediated device support +3. Nvidia mediated device support +4. AMDGPU mediated device support +5. YAML Configuration files +6. Create VM +7. Start VM +8. Stop VM +9. List available kernels +10. List available states +11. List running kernels +12. Deploy script +13. Undeploy script # Future Features -1. Intel GVT-g support -2. Snapshot features + Block diff copy -3. Hotplugging functionality +1. Wider OS support for automated installation +2. Snapshot + Block diff copy +3. Runtime hot-plugging/hot-unplugging # License diff --git a/example/amd-mdev-preinstall.yaml b/example/amd-mdev-preinstall.yaml new file mode 100644 index 0000000..2e9bdfb --- /dev/null +++ b/example/amd-mdev-preinstall.yaml @@ -0,0 +1,37 @@ +%YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: +--- !n!custom:Config +startintro: false +nographics: false +spice: false +introspect: "looking-glass" +shareddir: !!null ~ +connectivity: + exposedPorts: + - + guest: 22 + host: 2222 +container: + kernel: windows.arc + state: [] + initialSize: 20 + iso: !!null ~ +cpus: + cores: 8 + sockets: 1 + threads: 2 + ramAlloc: 8192 +gpus: + - + - + gpuType: "sriovdev" + - + acceptableTypes: [S7150] + - + maxVRam: 8192 + - + minVRam: 0 +nics: [] +root: /opt/arc +sudo: false +commands: [] diff --git a/example/amd-mdev.yaml b/example/amd-mdev.yaml index 9fe2885..37bb445 100644 --- a/example/amd-mdev.yaml +++ b/example/amd-mdev.yaml @@ -1,19 +1,18 @@ %YAML 1.2 %TAG !n! tag:nimyaml.org,2016: --- !n!custom:Config +startintro: true +nographics: true +spice: true +introspect: "looking-glass" +shareddir: !!null ~ connectivity: exposedPorts: - guest: 22 host: 2222 - - - guest: 5901 - host: 5900 - - - guest: 8080 - host: 8000 container: - kernel: "ubuntu-20.04.arc" + kernel: windows.arc state: [] initialSize: 20 iso: !!null ~ @@ -24,9 +23,14 @@ cpus: ramAlloc: 8192 gpus: - - acceptableTypes: [S7150] - maxVRam: 8000 - minVRam: 0 + - + gpuType: "sriovdev" + - + acceptableTypes: [S7150] + - + maxVRam: 8192 + - + minVRam: 0 nics: [] root: /opt/arc sudo: false diff --git a/example/intel-mdev-preinstall.yaml b/example/intel-mdev-preinstall.yaml new file mode 100644 index 0000000..d334ea6 --- /dev/null +++ b/example/intel-mdev-preinstall.yaml @@ -0,0 +1,37 @@ +%YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: +--- !n!custom:Config +startintro: false +nographics: false +spice: false +introspect: "looking-glass" +shareddir: !!null ~ +connectivity: + exposedPorts: + - + guest: 22 + host: 2222 +container: + kernel: windows.arc + state: [] + initialSize: 20 + iso: !!null ~ +cpus: + cores: 8 + sockets: 1 + threads: 2 + ramAlloc: 8192 +gpus: + - + - + gpuType: "sysfsdev" + - + parentPort: "0000:01:00.0" + - + mdevType: "i915-GVTg_V4_1" + - + devId: hostdev0 +nics: [] +root: /data/arc +sudo: false +commands: [] diff --git a/example/intel-mdev.yaml b/example/intel-mdev.yaml new file mode 100644 index 0000000..5312502 --- /dev/null +++ b/example/intel-mdev.yaml @@ -0,0 +1,37 @@ +%YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: +--- !n!custom:Config +startintro: true +nographics: true +spice: true +introspect: "looking-glass" +shareddir: !!null ~ +connectivity: + exposedPorts: + - + guest: 22 + host: 2222 +container: + kernel: windows.arc + state: [] + initialSize: 20 + iso: !!null ~ +cpus: + cores: 8 + sockets: 1 + threads: 2 + ramAlloc: 8192 +gpus: + - + - + gpuType: "sysfsdev" + - + parentPort: "0000:01:00.0" + - + mdevType: "i915-GVTg_V4_1" + - + devId: hostdev0 +nics: [] +root: /data/arc +sudo: false +commands: [] diff --git a/example/nvidia-mdev-preinstall.yaml b/example/nvidia-mdev-preinstall.yaml new file mode 100644 index 0000000..17ba021 --- /dev/null +++ b/example/nvidia-mdev-preinstall.yaml @@ -0,0 +1,37 @@ +%YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: +--- !n!custom:Config +startintro: false +nographics: false +spice: false +introspect: "looking-glass" +shareddir: !!null ~ +connectivity: + exposedPorts: + - + guest: 22 + host: 2222 +container: + kernel: windows.arc + state: [] + initialSize: 20 + iso: !!null ~ +cpus: + cores: 8 + sockets: 1 + threads: 2 + ramAlloc: 8192 +gpus: + - + - + gpuType: "sysfsdev" + - + parentPort: "0000:01:00.0" + - + mdevType: "nvidia-48" + - + devId: hostdev0 +nics: [] +root: /data/arc +sudo: false +commands: [] diff --git a/example/nvidia-mdev.yaml b/example/nvidia-mdev.yaml index d707587..1ae282d 100644 --- a/example/nvidia-mdev.yaml +++ b/example/nvidia-mdev.yaml @@ -1,19 +1,18 @@ %YAML 1.2 %TAG !n! tag:nimyaml.org,2016: --- !n!custom:Config +startintro: true +nographics: true +spice: true +introspect: "looking-glass" +shareddir: !!null ~ connectivity: exposedPorts: - guest: 22 host: 2222 - - - guest: 5901 - host: 5900 - - - guest: 8080 - host: 8000 container: - kernel: "ubuntu-20.04.arc" + kernel: windows.arc state: [] initialSize: 20 iso: !!null ~ @@ -22,15 +21,17 @@ cpus: sockets: 1 threads: 2 ramAlloc: 8192 -gpus: [] -nics: [] -root: /opt/arc -sudo: true -commands: +gpus: - - arg: "-device" - values: - - "vfio-pci" - - id=hostdev0 - - "sysfsdev=/sys/bus/mdev/devices/3ab19d97-d8a0-4617-8ab5-b3b1753f257d" - - display=off + - + gpuType: "sysfsdev" + - + parentPort: "0000:01:00.0" + - + mdevType: "nvidia-48" + - + devId: hostdev0 +nics: [] +root: /data/arc +sudo: false +commands: [] diff --git a/libvfio.nimble b/libvfio.nimble index 00c4a01..10d7634 100644 --- a/libvfio.nimble +++ b/libvfio.nimble @@ -1,6 +1,6 @@ # Package -version = "1.0.1" +version = "1.0.2" author = "2666680 Ontario Inc." description = "Release 1" license = "AGPL" diff --git a/optional/.gitkeep b/optional/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/patches/twelve.patch b/patches/twelve.patch new file mode 100644 index 0000000..655c68f --- /dev/null +++ b/patches/twelve.patch @@ -0,0 +1,264 @@ +diff -ur ./kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.c ../nv-patch/kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.c +--- ./kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.c 2021-06-28 11:17:00.000000000 -0400 ++++ ../nv-patch/kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.c 2021-10-07 15:10:03.070028263 -0400 +@@ -24,7 +24,6 @@ + #include + #include + #include +-#include + #include "nvstatus.h" + #include "nv-misc.h" + #include "nv-linux.h" +@@ -2667,18 +2666,19 @@ + + static int vgpu_save_fd(vgpu_dev_t *vgpu_dev, int fd, NvU32 index) + { +- struct eventfd_ctx *evt; ++ struct fd irqfd; + +- evt = eventfd_ctx_fdget(fd); +- if (IS_ERR(evt)) +- return PTR_ERR(evt); ++ irqfd = fdget(fd); ++ if (!irqfd.file) ++ return -EBADF; + + if (index == VFIO_PCI_INTX_IRQ_INDEX) +- vgpu_dev->intr_info.intx_evtfd = evt; +- else if (index == VFIO_PCI_MSI_IRQ_INDEX) +- vgpu_dev->intr_info.msi_evtfd = evt; ++ vgpu_dev->intr_info.intx_file = irqfd.file; ++ else if (index == VFIO_PCI_MSI_IRQ_INDEX) ++ vgpu_dev->intr_info.msi_file = irqfd.file; + + vgpu_dev->intr_info.index = index; ++ fdput(irqfd); + + return 0; + } +@@ -2687,8 +2687,11 @@ + static irqreturn_t vgpu_msix_handler(int irq, void *arg) + { + vgpu_dev_t *vgpu_dev = (vgpu_dev_t *)arg; +- struct eventfd_ctx *evt = NULL; ++ struct file *pfile = NULL; ++ mm_segment_t old_fs; ++ NvU64 val = 1; + int ret = 0; ++ loff_t offset = 0; + int i; + unsigned long eflags; + +@@ -2696,16 +2699,21 @@ + { + if (vgpu_dev->intr_info.allocated_irq[i] == irq) + { +- evt = vgpu_dev->intr_info.msix_evtfd[i]; ++ pfile = vgpu_dev->intr_info.msix_fd[i].file; + break; + } + } + +- if (evt) ++ if (pfile && pfile->f_op && pfile->f_op->write) + { ++ old_fs = get_fs(); ++ set_fs(KERNEL_DS); ++ + NV_SAVE_FLAGS(eflags); +- ret = eventfd_signal(evt, 1); ++ ret = pfile->f_op->write(pfile, (char *)&val, sizeof(val), &offset); + NV_RESTORE_FLAGS(eflags); ++ ++ set_fs(old_fs); + } + + return IRQ_HANDLED; +@@ -2716,24 +2724,23 @@ + { + struct pci_dev *pdev; + int irq = INVALID_IRQ, ret; +- struct eventfd_ctx *evt; ++ struct fd irqfd; + + pdev = to_pci_dev(NV_GET_MDEV_PARENT(vgpu_dev->mdev)); + +- if (vgpu_dev->intr_info.msix_evtfd[vector]) ++ if (vgpu_dev->intr_info.msix_fd[vector].file) + { + free_irq(vgpu_dev->intr_info.allocated_irq[vector], vgpu_dev); +- eventfd_ctx_put(vgpu_dev->intr_info.msix_evtfd[vector]); +- vgpu_dev->intr_info.msix_evtfd[vector] = NULL; ++ vgpu_dev->intr_info.msix_fd[vector].file = NULL; + vgpu_dev->intr_info.allocated_irq[vector] = INVALID_IRQ; + } + + if (fd < 0) + return 0; + +- evt = eventfd_ctx_fdget(fd); +- if (IS_ERR(evt)) +- return PTR_ERR(evt); ++ irqfd = fdget(fd); ++ if (!irqfd.file) ++ return -EBADF; + + if (vector < 0 || vector >= vgpu_dev->intr_info.num_ctx) + return -EINVAL; +@@ -2749,7 +2756,7 @@ + + vgpu_dev->intr_info.allocated_irq[vector] = irq; + +- vgpu_dev->intr_info.msix_evtfd[vector]= evt; ++ vgpu_dev->intr_info.msix_fd[vector]= irqfd; + + return 0; + } +@@ -2766,12 +2773,7 @@ + if (vgpu_dev->intr_info.allocated_irq[i] != INVALID_IRQ) + { + free_irq(vgpu_dev->intr_info.allocated_irq[i], vgpu_dev); +- +- if (vgpu_dev->intr_info.msix_evtfd[i]) { +- eventfd_ctx_put(vgpu_dev->intr_info.msix_evtfd[i]); +- vgpu_dev->intr_info.msix_evtfd[i] = NULL; +- } +- ++ vgpu_dev->intr_info.msix_fd[i].file = NULL; + vgpu_dev->intr_info.allocated_irq[i] = INVALID_IRQ; + } + } +@@ -2860,10 +2862,7 @@ + { + if (flags & VFIO_IRQ_SET_DATA_NONE) + { +- if (vgpu_dev->intr_info.intx_evtfd) { +- eventfd_ctx_put(vgpu_dev->intr_info.intx_evtfd); +- vgpu_dev->intr_info.intx_evtfd = NULL; +- } ++ vgpu_dev->intr_info.intx_file = NULL; + break; + } + +@@ -2888,10 +2887,7 @@ + { + if (flags & VFIO_IRQ_SET_DATA_NONE) + { +- if (vgpu_dev->intr_info.msi_evtfd) { +- eventfd_ctx_put(vgpu_dev->intr_info.msi_evtfd); +- vgpu_dev->intr_info.msi_evtfd = NULL; +- } ++ vgpu_dev->intr_info.msi_file = NULL; + vgpu_dev->intr_info.index = VFIO_PCI_INTX_IRQ_INDEX; + break; + } +@@ -2899,9 +2895,10 @@ + if (flags & VFIO_IRQ_SET_DATA_EVENTFD) + { + int fd = *(int *)data; +- if (fd > 0 && !vgpu_dev->intr_info.msi_evtfd) ++ if (fd > 0) + { +- ret = vgpu_save_fd(vgpu_dev, fd, index); ++ if (vgpu_dev->intr_info.msi_file == NULL) ++ ret = vgpu_save_fd(vgpu_dev, fd, index); + } + } + break; +@@ -2956,9 +2953,12 @@ + + NV_STATUS nv_vgpu_inject_interrupt(void *vgpuRef) + { ++ mm_segment_t old_fs; ++ NvU64 val = 1; + int ret = 0; ++ loff_t offset = 0; + NV_STATUS status = NV_OK; +- struct eventfd_ctx *evt = NULL; ++ struct file *pfile = NULL; + vgpu_dev_t *vgpu_dev = vgpuRef; + unsigned long eflags; + +@@ -2967,12 +2967,12 @@ + + NV_SPIN_LOCK_IRQSAVE(&vgpu_dev->intr_info_lock, eflags); + +- if ((vgpu_dev->intr_info.index == VFIO_PCI_MSI_IRQ_INDEX) && (!vgpu_dev->intr_info.msi_evtfd)) ++ if ((vgpu_dev->intr_info.index == VFIO_PCI_MSI_IRQ_INDEX) && (vgpu_dev->intr_info.msi_file == NULL)) + { + NV_SPIN_UNLOCK_IRQRESTORE(&vgpu_dev->intr_info_lock, eflags); + return NV_ERR_INVALID_REQUEST; + } +- else if ((vgpu_dev->intr_info.index == VFIO_PCI_INTX_IRQ_INDEX) && (!vgpu_dev->intr_info.intx_evtfd)) ++ else if ((vgpu_dev->intr_info.index == VFIO_PCI_INTX_IRQ_INDEX) && (vgpu_dev->intr_info.intx_file == NULL)) + { + NV_SPIN_UNLOCK_IRQRESTORE(&vgpu_dev->intr_info_lock, eflags); + return NV_ERR_INVALID_REQUEST; +@@ -2984,9 +2984,9 @@ + } + + if (vgpu_dev->intr_info.index == VFIO_PCI_MSI_IRQ_INDEX) +- evt = vgpu_dev->intr_info.msi_evtfd; ++ pfile = vgpu_dev->intr_info.msi_file; + else +- evt = vgpu_dev->intr_info.intx_evtfd; ++ pfile = vgpu_dev->intr_info.intx_file; + + // QEMU has exited. So, safe to ignore interrupts. + if (vgpu_dev->intr_info.ignore_interrupts == NV_TRUE) +@@ -2996,14 +2996,19 @@ + } + NV_SPIN_UNLOCK_IRQRESTORE(&vgpu_dev->intr_info_lock, eflags); + +- if (evt) +- ret = eventfd_signal(evt, 1); +- else +- status = NV_ERR_INVALID_REQUEST; ++ old_fs = get_fs(); ++ set_fs(KERNEL_DS); ++ ++ if (pfile->f_op && pfile->f_op->write) ++ ret = pfile->f_op->write(pfile, (char *)&val, sizeof(val), &offset); ++ else ++ status = NV_ERR_INVALID_REQUEST; + + if (ret < 0) + status = NV_ERR_INVALID_STATE; + ++ set_fs(old_fs); ++ + return status; + } + +Only in ../nv-patch/kernel/nvidia-vgpu-vfio: nvidia-vgpu-vfio.c.orig +diff -ur ./kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.h ../nv-patch/kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.h +--- ./kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.h 2021-06-28 11:16:11.000000000 -0400 ++++ ../nv-patch/kernel/nvidia-vgpu-vfio/nvidia-vgpu-vfio.h 2021-10-07 15:10:04.210028588 -0400 +@@ -37,7 +37,6 @@ + #include + #include + #include +-#include + + struct vgpu_dev_s; + struct mapping_node_s; +@@ -294,15 +293,15 @@ + + typedef struct + { +- struct eventfd_ctx *intx_evtfd; +- struct eventfd_ctx *msi_evtfd; ++ struct file *intx_file; ++ struct file *msi_file; + int index; + NvBool ignore_interrupts; + + NvU32 allocated_irq[MAX_NUM_VECTORS]; + NvU32 num_ctx; + #if defined(NV_VGPU_KVM_BUILD) +- struct eventfd_ctx *msix_evtfd[MAX_NUM_VECTORS]; ++ struct fd msix_fd[MAX_NUM_VECTORS]; + #endif + + } intr_info_t; +Only in ../nv-patch/kernel/nvidia-vgpu-vfio: nvidia-vgpu-vfio.h.orig diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..fae8d4e --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Copyright: 2666680 Ontario Inc. +# Reason: Installation of libvf.io +# + +# Place optional driver packages in the optional directory before running this installation script + +currentPath=$(pwd) +# Compile sandbox path +compileSandbox=$(echo ~)"/.cache/libvf.io/compile/" + +if [ ! -f "$HOME/preinstall" ]; then + +sudo usermod -a -G kvm $USER + +# Configure kernel boot parameters +echo "Updating kernel boot parameters." +# Intel users +sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT=\"quiet splash\"/GRUB_CMDLINE_LINUX_DEFAULT=\"intel_iommu=on iommu=pt vfio_pci\"/g' /etc/default/grub +sudo update-grub + + +# Configure AppArmor policies, shared memory file permissions, and blacklisting non-mediated device drivers. +echo "Updating AppArmor policies." +sudo su root -c "mkdir -p /etc/apparmor.d/local/abstractions/ && echo '/dev/shm/kvmfr-* rw,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu" +echo "Configuring shared memory device file permissions." +sudo su root -c "echo 'f /dev/shm/kvmfr-* 0660 user kvm -' >> /etc/tmpfiles.d/10-looking-glass.conf" +echo "Blacklisting non-mediated device drivers." +sudo su root -c "echo '# Libvf.io GPU driver blacklist' >> /etc/modprobe.d/blacklist.conf && echo 'blacklist nouveau' >> /etc/modprobe.d/blacklist.conf && echo 'blacklist amdgpu' >> /etc/modprobe.d/blacklist.conf && echo 'blacklist amdkfd' >> /etc/modprobe.d/blacklist.conf" +# Restarting apparmor service +sudo systemctl restart apparmor +# Updating initramfs +sudo update-initramfs -u -k all +# rmmod nouveau +sudo rmmod nouveau + + +# Install base utilities and build dependencies +sudo apt install -y dkms libglvnd-dev curl gcc cmake fonts-freefont-ttf libegl-dev libgl-dev libfontconfig1-dev libgmp-dev libspice-protocol-dev make nettle-dev pkg-config python3 python3-pip binutils-dev qemu qemu-utils qemu-kvm libx11-dev libxfixes-dev libxi-dev libxinerama-dev libxss-dev libwayland-bin libwayland-dev wayland-protocols gcc-mingw-w64-x86-64 nsis mdevctl git + +# Install choosenim +curl https://nim-lang.org/choosenim/init.sh -sSf | sh +echo "export PATH=$HOME/.nimble/bin:$PATH" >> ~/.bashrc +export PATH=$HOME/.nimble/bin:$PATH +choosenim update stable + + +# Compile and install libvf.io +cd $currentPath +nimble install -y +rm ./arcd + +# Download Looking Glass beta 4 sources +mkdir -p $compileSandbox +cd $compileSandbox +rm -rf LookingGlass +git clone --recursive https://github.com/gnif/LookingGlass/ +cd LookingGlass +git checkout Release/B4 + +# Compile & install Looking Glass sources +mkdir client/build +mkdir host/build +cd client/build +cmake ../ +make +sudo make install + +# Cause we cannot use looking glass host binary +cd ../../host/build +cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-mingw64.cmake .. +make +cd platform/Windows +makensis installer.nsi + +rm -rf $HOME/.config/arc/introspection-installations +mkdir -p $HOME/.config/arc/introspection-installations +wget https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/upstream-virtio/virtio-win10-prewhql-0.1-161.zip +cp -r * $HOME/.config/arc/introspection-installations +cd $HOME/.config/arc/ +mkisofs -A introspection-installations.rom -l -allow-leading-dots -allow-limited-size -allow-lowercase -allow-multidot -relaxed-filenames -d -D -o ./introspection-installations.rom introspection-installations + +fi + +if [ ! -f $currentPath/optional/*.run ]; then + echo "Optional drivers not found." + exit 0 +fi + +cd $currentPath/optional + +kernel_release=$(uname -r) +major=`awk '{split($0, a, "."); print a[1]}' <<< $kernel_release` +minor=`awk '{split($0, a, "."); print a[2]}' <<< $kernel_release` + +echo "MAJOR: $major" +echo "MINOR: $minor" + +custom="" +if [[ ($major -eq 5) && ($minor -ge 12) ]];then + echo "Modifying the driver to have the version 5.12 patches." + custom="-custom" + ./*.run --apply-patch $currentPath/patches/twelve.patch +fi + +lsmod | grep "nouveau" + +if [ $? -ne 0 ]; then + openssl req -new -x509 -newkey rsa:4096 -keyout ~/.ssh/module-private.key -outform DER -out ~/.ssh/module-public.key -nodes -days 3650 -subj "/CN=kernel-module" + echo "The following password will need to be used in enroll MOK on your next startup." + sudo mokutil --import ~/.ssh/module-public.key + sudo ./*$custom.run --module-signing-secret-key=$HOME/.ssh/module-private.key --module-signing-public-key=$HOME/.ssh/module-public.key -q + rm $HOME/preinstall +else + touch $HOME/preinstall + echo "Nouveau was found, please reboot and run ./install.sh again, it will start from this point." +fi + +sudo systemctl daemon-reload diff --git a/src/arcd.nim b/src/arcd.nim index a4e2c24..eaf402b 100644 --- a/src/arcd.nim +++ b/src/arcd.nim @@ -5,7 +5,7 @@ import options import os -import libvfiopkg/[control, logger, types, comms] +import libvfio/[control, logger, types, comms] when isMainModule: let @@ -26,6 +26,8 @@ when isMainModule: startVm(cfg, uid, false, cmd.nocopy, cmd.save) of ceStop: stopVm(cfg, cmd) + of ceIntrospect: + introspectVm(cfg, cmd.uuid) of ceLs: arcLs(cfg, cmd) of cePs: @@ -33,10 +35,13 @@ when isMainModule: of ceDeploy: removeFile(homeConfigDir / "arc.yaml") writeConfigFile(homeConfigDir / "arc.yaml", cfg) - createDir(cfg.root) + createDir(cfg.root / "states") + copyFile( + homeConfigDir / "introspection-installations.rom", + cfg.root / "introspection-installations.rom" + ) of ceUndeploy: removeFile(homeConfigDir / "arc.yaml") - removeDir(homeConfigDir) removeDir(cfg.root) if cmd.save and isSome(cmd.config): diff --git a/src/libvfiopkg/comms.nim b/src/libvfio/comms.nim similarity index 100% rename from src/libvfiopkg/comms.nim rename to src/libvfio/comms.nim diff --git a/src/libvfiopkg/comms/ls.nim b/src/libvfio/comms/ls.nim similarity index 100% rename from src/libvfiopkg/comms/ls.nim rename to src/libvfio/comms/ls.nim diff --git a/src/libvfiopkg/comms/ps.nim b/src/libvfio/comms/ps.nim similarity index 100% rename from src/libvfiopkg/comms/ps.nim rename to src/libvfio/comms/ps.nim diff --git a/src/libvfiopkg/comms/qmp.nim b/src/libvfio/comms/qmp.nim similarity index 100% rename from src/libvfiopkg/comms/qmp.nim rename to src/libvfio/comms/qmp.nim diff --git a/src/libvfiopkg/control.nim b/src/libvfio/control.nim similarity index 69% rename from src/libvfiopkg/control.nim rename to src/libvfio/control.nim index 9a2a16f..ac03ed8 100644 --- a/src/libvfiopkg/control.nim +++ b/src/libvfio/control.nim @@ -2,6 +2,8 @@ # Copyright: 2666680 Ontario Inc. # Reason: Reexports the control for libvfio. # +import control/introspection import control/vm +export introspection export vm diff --git a/src/libvfio/control/arguments.nim b/src/libvfio/control/arguments.nim new file mode 100644 index 0000000..fa5e0de --- /dev/null +++ b/src/libvfio/control/arguments.nim @@ -0,0 +1,381 @@ +# +# Copyright: 2666680 Ontario Inc. +# Reason: Creates commands to execute on the host system. +# +import strformat +import strutils +import sequtils +import sugar +import options +import os + +import ../types + +const + # Additional strings to help hide KVM status from the resources. + MachineConfig = join( + @["pc-q35-4.2", + "accel=kvm", + "usb=off", + "vmport=off", + "dump-guest-core=off" + ], + "," + ) + CpuConfig = join( + @["IvyBridge-IBRS", + "ss=on", + "vmx=on", + "pcid=on", + "hypervisor=on", + "arat=on", + "tsc-adjust=on", + "umip=on", + "md-clear=on", + "stibp=on", + "arch-capabilities=on", + "ssbd=on", + "xsaveopt=on", + "pdpe1gb=on", + "ibpb=on", + "ibrs=on", + "amd-stibp=on", + "amd-ssbd=on", + "skip-l1dfl-vmentry=on", + "pschange-mc-no=on", + "hv-vapic", + "hv-spinlocks=0x1fff", + "hv-vendor-id=1234567890ab", + "kvm=off" + ], + "," + ) + +proc runCommand*(command: Args): bool = + ## runCommand - Helper to run a command. + ## + ## Inputs + ## @command - Command to run. + ## + ## Returns + ## result - If the command was successful or not. + ## + ## Side Effects - Arbitrarily executes the command and checks return status. + execShellCmd(join(@[command.exec] & command.args, " ")) == 0 + +func removeFiles*(files: seq[string]): Args = + ## removeFiles - Mass removes a set of files using sudo. + ## + ## Inputs + ## @files - List of files to remove + ## + ## Returns + ## result - Argument to run to remove a set of files using sudo. + result.exec = "/usr/bin/sudo" + result.args &= "/bin/rm" + result.args &= "-rf" + result.args &= files + +func startMdev*(uuid: string, parent: string, mdevType: string): Args = + ## startMdev - Starts an mdevctl device. + ## + ## Inputs + ## @uuid - The UUID of the device. + ## @parent - The parent of the device. + ## @mdevType - Type of mediated device to make. + ## + ## Returns + ## result - Argument to run to start a mediated device. + ## + ## NOTE: Can only be run as sudo. + ## + ## TODO: Replace with our own system. + result.exec = "/usr/bin/sudo" + result.args &= "/usr/sbin/mdevctl" + result.args &= "start" + result.args &= "-u" + result.args &= uuid + result.args &= "-p" + result.args &= parent + result.args &= "-t" + result.args &= mdevType + +func stopMdev*(uuid: string): Args = + ## stopMdev - Stops an mdevctl device. + ## + ## Inputs + ## @uuid - The UUID of the device. + ## + ## Returns + ## result - Argument to run to stop a mediated device. + ## + ## NOTE: Can only be run as sudo. + ## + ## TODO: Replace with our own system. + result.exec = "/usr/bin/sudo" + result.args &= "/usr/sbin/mdevctl" + result.args &= "stop" + result.args &= "-u" + result.args &= uuid + +func createKernel*(name: string, size: int): Args = + ## createKernel - Creates a kernel image for the Arc Container. + ## + ## Inputs + ## @name - Name of the kernel image. + ## @size - Initial size of the container. + ## + ## Returns + ## result - Arguments to create a kernel. + result.exec = "qemu-img" + result.args &= "create" + result.args &= "-f" + result.args &= "qcow2" + result.args &= name + result.args &= &"{size}G" + +func changeGroup*(sudo: bool, files: seq[string]): Args = + ## changeGroup - Changes the files group to allow not to use sudo. + ## + ## Inputs + ## @sudo - Do we use sudo? + ## @files - Files to change. + ## + ## Returns + ## result - Arguments to change the file group. + result.exec = "/usr/bin/chgrp" + + # If we need sudo + if sudo: + result.args &= result.exec + result.exec = "/bin/sudo" + + # Set the group to KVM + result.args &= "kvm" + + # Add all files + result.args &= files + +func changePermissions*(sudo: bool, files: seq[string]): Args = + ## changePermissions - Changes the permission on the file. + ## + ## Inputs + ## @sudo - Do we use sudo? + ## @files - Files to change. + ## + ## Returns + ## result - Arguments to change the permissions for the files. + result.exec = "/usr/bin/chmod" + + # If we need sudo + if sudo: + result.args &= result.exec + result.exec = "/bin/sudo" + + # Set the group to KVM + result.args &= "g+rwx" + + # Add all files + result.args &= files + + +func qemuLaunch*(cfg: Config, uuid: string, + vfios: seq[Vfio], mdevs: seq[Mdev], + kernel: string, install: bool, + logDir: string, sockets: seq[string]): Args = + ## qemuLaunch - Generates a qemu command to be executed. + ## + ## Inputs + ## @cfg - Configuration for the qemu launch. + ## @uuid - UUID of the app. + ## @vfios - List of VFIOs to put pass into the VM. + ## @mdevs - List of MDevs to pass into the VM. + ## @kernel - Kernel path. + ## @install - Is this an installation? + ## @logDir - Directory for logging. + ## @sockets - Sockets to create. + ## + ## Returns + ## result - Arguments to launch a kernel. + func vfioArgs(device: Vfio): seq[string] = + if isGpu(device): + result &= "-device" + result &= &"vfio-pci,host={device},multifunction=on,display=off" + result &= "-mem-prealloc" + + func mdevArgs(device: Mdev): seq[string] = + const mdevBase = "/sys/bus/mdev/devices" + result &= "-device" + result &= + &"vfio-pci,id={device.devId},sysfsdev={mdevBase}/{device.uuid},display=off" + + let + qemuLogFile = logDir / (uuid & "-session.txt") + + # executing target + result.exec = "/bin/qemu-system-x86_64" + + # If we need sudo + if cfg.sudo: + result.args &= result.exec + result.exec = "/bin/sudo" + + # Log file + result.args &= "-D" + result.args &= qemuLogFile + + # Causes issues with mdev devices. + result.args &= "-no-hpet" + + # NOTE: We do not allow installing in no graphics mode just yet. + if cfg.nographics and not install: + # No graphics flag + result.args &= "-nographic" + + # Sets the VGA output to nothing + result.args &= "-vga" + result.args &= "none" + + # If spice server is enabled. + if cfg.spice and not install: + # Additional no defaults flags which cause issues with MDev support + result.args &= "-serial" + result.args &= "none" + + result.args &= "-parallel" + result.args &= "none" + + # Additional mouse for USB redirects from spice + result.args &= "-device" + result.args &= "qemu-xhci,p2=15,p3=15,id=usb" + + # VirtIO Serial PCI Device + result.args &= "-device" + result.args &= "virtio-serial-pci,id=virtio-serial0" + + # Chardev + result.args &= "-chardev" + result.args &= "pty,id=charserial0" + + # ISA Serial device + result.args &= "-device" + result.args &= "isa-serial,chardev=charserial0,id=serial0" + + # SPICE Char Channel + result.args &= "-chardev" + result.args &= "spicevmc,id=charchannel0,name=vdagent" + + # Virtual Serial Port + result.args &= "-device" + result.args &= + "virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0" + + # Spice port + result.args &= "-spice" + result.args &= + "port=5900,addr=127.0.0.1,disable-ticketing,image-compression=off,seamless-migration=on" + + # SPICE USB Redirects + result.args &= "-chardev" + result.args &= "spicevmc,id=charredir0,name=usbredir" + result.args &= "-device" + result.args &= "usb-redir,chardev=charredir0,id=redir0" + result.args &= "-chardev" + result.args &= "spicevmc,id=charredir1,name=usbredir" + result.args &= "-device" + result.args &= "usb-redir,chardev=charredir1,id=redir1" + + # Introspection related commands. + # NOTE: All introspection devices cannot be used on install + if not install: + case cfg.introspect + of isLookingGlass: + # Create device for looking glass + result.args &= "-device" + result.args &= "ivshmem-plain,id=shmem0,memdev=ivshmem" + + # Create looking glass object + result.args &= "-object" + result.args &= + &"memory-backend-file,id=ivshmem,mem-path=/dev/shm/kvmfr-{uuid},size=128M,share=yes" + else: discard + + # UUID - Necessary for NVidia MDEV support + result.args &= "-uuid" + result.args &= uuid + + # Machine settings needed for NVidia MDEV support + result.args &= "-machine" + result.args &= MachineConfig + + # Cpu settings needed for NVidia MDEV support + result.args &= "-cpu" + result.args &= CpuConfig + + # clock settings + result.args &= "-rtc" + result.args &= "clock=host,base=localtime" + + # RAM allocation + result.args &= "-m" + result.args &= $cfg.cpus.ramAlloc + + # CPU configuration + result.args &= "-smp" + result.args &= $cfg.cpus + + # Kernel path + result.args &= "-hda" + result.args &= kernel + + # Enable KVM + result.args &= "--enable-kvm" + + # Set vfio arguments + for vfio in vfios: + result.args &= vfioArgs(vfio) + + # Set mdev arguments + for mdev in mdevs: + result.args &= mdevArgs(mdev) + + # Enable sound + result.args &= "--soundhw" + result.args &= "all" + + # Port forward for all exposed ports + if len(cfg.connectivity.exposedPorts) > 0: + let hostfwds = map(cfg.connectivity.exposedPorts, + (port: Port) => &"hostfwd=tcp::{port.host}-:{port.guest}") + result.args &= "-device" + result.args &= "rtl8139,netdev=net0" + result.args &= "-netdev" + result.args &= join(@["user", "id=net0"] & hostfwds, ",") + + if isSome(cfg.shareddir) and not install: + result.args &= "-hdb" + result.args &= &"fat:rw:{get(cfg.shareddir)}" + + # Enable QMP socket for all sockets. + for socket in sockets: + result.args &= "-qmp" + result.args &= &"unix:{socket},server,nowait" + + # States + for i, state in cfg.container.state: + let s = cfg.root / "states" / state + result.args &= "-hdd" + result.args &= s + + # If it is being installed + if isSome(cfg.container.iso): + result.args &= "-cdrom" + result.args &= get(cfg.container.iso) + + # Additional commands sent into the qemu command + # NOTE: Start Process quotes these commands. + for command in cfg.commands: + let values = join(command.values, ",") + result.args &= command.arg + if values != "": + result.args &= values diff --git a/src/libvfio/control/introspection.nim b/src/libvfio/control/introspection.nim new file mode 100644 index 0000000..4a4a34a --- /dev/null +++ b/src/libvfio/control/introspection.nim @@ -0,0 +1,96 @@ +# +# Copyright: 2666680 Ontario Inc. +# Reason: Introspection specific code. +# +import os +import osproc +import posix +import strformat +import logging + +import ../types + +func getIntrospections*(cfg: Config, uuid: string, + install: bool = false): seq[string] = + ## getIntrospections - Gets a list of introspection devices. + ## + ## Inputs + ## @cfg - Configuration file to use. + ## @uuid - UUID to use. + ## @install - If we are installing there are no introspection tools we can use. + ## + ## Returns + ## result - A list of devices that can be used for introspection. + if not install: + case cfg.introspect + of isLookingGlass: + result = @["/dev/shm/kvmfr-" & uuid] + else: discard + +proc lookingGlassIntrospect(introspections: seq[string], uuid: string) = + ## lookingGlassIntrospect - Introspection using looking glass. + ## + ## Inputs + ## @introspections - Devices we can run introspections on. + ## @uuid - UUID Name for this introspection. + ## + ## Side effect - Spawns up a looking glass introspection window. + let + lookingGlassArgs = Args( + exec: "/usr/local/bin/looking-glass-client", + args: @[ + "-f", introspections[0], "win:autoResize", + "yes", "input:mouseRedraw", "yes", "win:keepAspect", + "yes", "egl:vsync", "yes", "egl:scale", "1", "win:title", + uuid + ] + ) + + # Fork to spawn up the introspection client. + let forkRet = fork() + if forkRet > 0: + return + elif forkRet < 0: + error("Could not fork for introspection process") + return + + # Spawn up looking glass + var + lookingGlassPid = startProcess( + lookingGlassArgs.exec, + args=lookingGlassArgs.args, + options={poEchoCmd, poParentStreams} + ) + + # This allows us to potentially expend this introspection. + discard waitForExit(lookingGlassPid) + quit(0) + +proc realIntrospect*(intro: IntrospectEnum, introspections: seq[string], + uuid: string) = + ## introspectVm - Starts an introspection script for the VM. + ## + ## Inputs + ## @intro - Introspection type. + ## @introspections - List of introspection devices. + ## @uuid - UUID for the name of the introspection. + ## + ## Side effects - Opens all introspection devices. + case intro + of isLookingGlass: + lookingGlassIntrospect(introspections, uuid) + else: discard + +proc introspectVm*(cfg: Config, uuid: string) = + ## introspectVm - Starts an introspection script for the VM. + ## + ## Inputs + ## @cfg - Configuration file to use. + ## @uuid - UUID of the VM to introspect. + ## + ## Side effects - Opens all introspection devices. + let + lockFile = cfg.root / "lock" / &"{uuid}.json" + lock = getLockFile(lockFile) + config = lock.config + realIntrospect(config.introspect, getIntrospections(config, uuid), uuid) diff --git a/src/libvfiopkg/control/iommu.nim b/src/libvfio/control/iommu.nim similarity index 83% rename from src/libvfiopkg/control/iommu.nim rename to src/libvfio/control/iommu.nim index 6b9259a..73c0717 100644 --- a/src/libvfiopkg/control/iommu.nim +++ b/src/libvfio/control/iommu.nim @@ -12,6 +12,8 @@ import options import os import tables +import arguments + import ../types type @@ -222,15 +224,56 @@ proc getVfios*(cfg: Config, uuid: string): seq[Vfio] = # How we select the correct GPUs. for i in requestedGpus: - for k, v in gpus: - var newSorted = v + case i.gpuType + of rgSRIOVGpu: + for k, v in gpus: + var newSorted = v + + sort(newSorted, gpuSort) + + let + (selectedGpu, newV) = selectGpu(i, newSorted) - sort(newSorted, gpuSort) + gpus[k] = newV + if isSome(selectedGpu): + result &= get(selectedGpu) + break + else: discard +proc getMdevs*(cfg: Config): seq[Mdev] = + ## getMdevs - Gets the list of MDEVs that need to be passed to the VM. + ## + ## Inputs + ## @cfg - Configuration file to use. + ## + ## Returns + ## result - List of selected MDEVs for the device in question. + ## + ## Side effects - Generates MDevs. + for i in cfg.gpus: + case i.gpuType + of rgMdevGpu: let - (selectedGpu, newV) = selectGpu(i, newSorted) + gpuUUID = getUUID() + startArgs = startMdev(gpuUUID, i.parentPort, i.mdevType) - gpus[k] = newV - if isSome(selectedGpu): - result &= get(selectedGpu) - break + if runCommand(startArgs): + result &= Mdev( + uuid: gpuUUID, + devId: i.devId + ) + else: discard + +proc getIommuGroups*(cfg: Config, uuid: string): (seq[Vfio], seq[Mdev]) = + ## getIommuGroups - Gets a list of vfios, and a seperate list of mdevs. + ## + ## Inputs + ## @cfg - Configuration file to use. + ## @uuid - Process's UUID. + ## + ## Returns + ## result - A list of vfios and a list of mdevs + let + vfios = getVfios(cfg, uuid) + mdevs = getMdevs(cfg) + result = (vfios, mdevs) diff --git a/src/libvfiopkg/control/vm.nim b/src/libvfio/control/vm.nim similarity index 77% rename from src/libvfiopkg/control/vm.nim rename to src/libvfio/control/vm.nim index 4eb1354..7db0809 100644 --- a/src/libvfiopkg/control/vm.nim +++ b/src/libvfio/control/vm.nim @@ -8,20 +8,21 @@ import os import osproc import posix import options -import strutils import strformat import sequtils import sugar import logging import arguments +import introspection import iommu import ../comms/qmp import ../types proc realCleanup(lockFile: string, uuid: string, socketDir: string, - vfios: seq[Vfio]) = + vfios: seq[Vfio], mdevs: seq[Mdev], + introspections: seq[string], sudo: bool) = ## realCleanup - Real cleanup function. ## ## Inputs @@ -29,6 +30,9 @@ proc realCleanup(lockFile: string, uuid: string, socketDir: string, ## @uuid - UUID for the VM. ## @socketDir - Socket directory for the system. ## @vfios - List of connected VFIOs. + ## @mdevs - List of connected MDevs. + ## @introspections - Introspected devices. + ## @sudo - Sudo status required or not. ## ## Side effects - Cleans up the VM. info("Cleaning up VM") @@ -47,11 +51,24 @@ proc realCleanup(lockFile: string, uuid: string, socketDir: string, discard info(&"Unlocked: {vfio.deviceName}") + + # Unlock all locked MDevs + for mdev in mdevs: + discard runCommand(stopMdev(mdev.uuid)) + info(&"Deleted MDEV: {mdev.devId}") + + if sudo: + discard runCommand(removeFiles(introspections)) + else: + for i in introspections: + removeFile(i) + info("Cleaned up VM") proc cleanupVm*(socket: AsyncSocket, pid: Process, lockFile: string, socketDir: string, - uuid: string, vfios: seq[Vfio]) = + uuid: string, vfios: seq[Vfio], mdevs: seq[Mdev], + introspections: seq[string], sudo: bool) = ## cleanupVm - Cleans up the entire VM active stack once it is finished ## executing. ## @@ -62,6 +79,9 @@ proc cleanupVm*(socket: AsyncSocket, pid: Process, ## @socketDir - Directory for where the sockets are stored. ## @uuid - UUID for the program. ## @vfios - List of locked VFIOs. + ## @mdevs - List of connected MDevs. + ## @introspections - Introspected devices. + ## @sudo - Sudo status required or not. ## ## Side effects - Deletes files, and sockets, also will kill the VM if ## necessary. @@ -76,7 +96,7 @@ proc cleanupVm*(socket: AsyncSocket, pid: Process, # before killing the process. timeouts += 1 elif poweringDown: # KILL THE PROCESS - terminate(pid) + terminate(pid) # TODO: Test with sudo waitFor(sleepAsync(300)) continue @@ -99,7 +119,7 @@ proc cleanupVm*(socket: AsyncSocket, pid: Process, ) else: discard - realCleanup(lockFile, uuid, socketDir, vfios) + realCleanup(lockFile, uuid, socketDir, vfios, mdevs, introspections, sudo) proc startVm*(c: Config, uuid: string, newInstall: bool, noCopy: bool, save: bool) = @@ -131,10 +151,11 @@ proc startVm*(c: Config, uuid: string, newInstall: bool, @["main.sock", "master.sock"], (s: string) => socketDir / s ) + introspections = getIntrospections(cfg, uuid, newInstall) # If we are passing a vfio, we need to run the command as sudo - let vfios = getVfios(cfg, uuid) - if len(vfios) > 0: + let (vfios, mdevs) = getIommuGroups(cfg, uuid) + if len(vfios) > 0 or len(mdevs) > 0: cfg.sudo = true # If we do not have the necessary directories, create them @@ -146,6 +167,7 @@ proc startVm*(c: Config, uuid: string, newInstall: bool, lock = Lock( config: cfg, vfios: vfios, + mdevs: mdevs, pidNum: 0 ) @@ -178,6 +200,7 @@ proc startVm*(c: Config, uuid: string, newInstall: bool, cfg=cfg, uuid=uuid, vfios=lock.vfios, + mdevs=lock.mdevs, kernel=liveKernel, install=newInstall, logDir=qemuLogs, @@ -198,22 +221,29 @@ proc startVm*(c: Config, uuid: string, newInstall: bool, sleep(3000) # Sleeping to avoid trying to open the file too soon. let - socketGroupArgs = changeSocketGroup(cfg.sudo, sockets) - permissionsArgs = changePermissions(cfg.sudo, sockets) + ownedFiles = sockets & introspections + socketGroupArgs = changeGroup(cfg.sudo, ownedFiles) + permissionsArgs = changePermissions(cfg.sudo, ownedFiles) # If we fail to change socket stuff - if not runCommand(socketGroupArgs) or not runCommand(permissionsArgs): + if cfg.sudo and + (not runCommand(socketGroupArgs) or not runCommand(permissionsArgs)): error("Could not change socket information correctly cleaning up.") - realCleanup(lockFile, uuid, socketDir, vfios) + realCleanup(lockFile, uuid, socketDir, vfios, mdevs, introspections, cfg.sudo) if not noCopy: removeFile(liveKernel) + return + + if cfg.startintro and not newInstall: + realIntrospect(cfg.introspect, introspections, uuid) let socketMaybe = createSocket(socketDir / "main.sock") socket = if isSome(socketMaybe): get(socketMaybe) else: newAsyncSocket() - cleanupVm(socket, qemuPid, lockFile, socketDir, uuid, vfios) + cleanupVm(socket, qemuPid, lockFile, socketDir, uuid, vfios, mdevs, + introspections, cfg.sudo) if newInstall or save: info("Installing to base kernel") diff --git a/src/libvfiopkg/logger.nim b/src/libvfio/logger.nim similarity index 100% rename from src/libvfiopkg/logger.nim rename to src/libvfio/logger.nim diff --git a/src/libvfiopkg/types.nim b/src/libvfio/types.nim similarity index 100% rename from src/libvfiopkg/types.nim rename to src/libvfio/types.nim diff --git a/src/libvfiopkg/types/config.nim b/src/libvfio/types/config.nim similarity index 76% rename from src/libvfiopkg/types/config.nim rename to src/libvfio/types/config.nim index abf07fc..c910563 100644 --- a/src/libvfiopkg/types/config.nim +++ b/src/libvfio/types/config.nim @@ -7,30 +7,52 @@ import options import os import streams import strutils +import logging import yaml import connectivity, hardware, environment, process type CommandEnum* = enum ## Different first layer commands. - ceCreate = "create", + ceLs = "ls", ceStart = "start", + ceCreate = "create", + ceIntrospect = "introspect" ceStop = "stop", - ceLs = "ls", cePs = "ps", ceDeploy = "deploy", ceUndeploy = "undeploy" + IntrospectEnum* = enum ## Available introspection tools + isNone = "none", + isLookingGlass = "looking-glass" + + RequestedGpuType* = enum ## Types of GPUs we support. + rgSRIOVGpu = "sriovdev", + rgMdevGpu = "sysfsdev" + RequestedGpu* = object ## Object to request a GPU. - acceptableTypes*: seq[string] ## Possible types of GPUs we accept - maxVRam*: int ## Maximum acceptable vRAM. - minVRam*: int ## Minimal acceptable vRAM. + case gpuType*: RequestedGpuType + of rgSRIOVGpu: + acceptableTypes*: seq[string] ## Possible types of GPUs we accept + maxVRam*: int ## Maximum acceptable vRAM. + minVRam*: int ## Minimal acceptable vRAM. + of rgMdevGpu: + parentPort*: string ## Requested parent port. + mdevType*: string ## Type of mediated device. + devId*: string ## Name of the device for additional commands + ## in the command argument. RequestedNet* = object ## Object to request a network NIC. mac*: string ## MAC address for the requested NIC. Config* = object ## Configuration object for spawning a ## VM. + startintro*: bool ## If we start the introspection by default. + nographics*: bool ## If we have the no graphics flag set. + spice*: bool ## If we want to use a spice server. + introspect*: IntrospectEnum ## What type of introspection we use. + shareddir*: Option[string] ## Shared directory between os and host. connectivity*: Connectivity ## Code to connect to the machine. container*: ArcContainer ## The specifics for how to spawn the ## container. @@ -49,6 +71,7 @@ type ## every single time (useful for prototyping) noconfig*: bool ## No user config preload. kernel*: Option[string] ## Kernel image to use. + shareddir*: Option[string] ## Shared directory. case command*: CommandEnum ## Different commands have different ## variables, so we need to only allow ## some variables to be used in the @@ -58,7 +81,7 @@ type size*: Option[int] ## Size of the initial kernel. of ceStart: additionalStates*: seq[string] ## Additional state variables to send in. - of ceStop: + of ceStop, ceIntrospect: uuid*: string ## UUID of the container we want to stop. of ceLs: option*: Option[string] ## Option for ls [all, kernels, states, apps] @@ -70,15 +93,18 @@ type const DefaultConfig = Config( ## Default configuration value if nothing ## is already found. + startintro: false, + nographics: false, + spice: false, + introspect: isLookingGlass, + shareddir: none(string), connectivity: Connectivity( exposedPorts: @[ Port(guest: 22, host: 2222), - Port(guest: 5901, host: 5900), - Port(guest: 8080, host: 8000) ] ), container: ArcContainer( - kernel: "ubuntu-20.04.arc", + kernel: "windows.arc", state: @[], initialSize: 20, iso: none(string) @@ -91,7 +117,7 @@ const ), gpus: @[], nics: @[], - root: "/opt/arc", + root: "/data/arc", sudo: false, commands: @[] ) @@ -106,6 +132,8 @@ proc getCommandLine*(): CommandLineArguments = ## Side effects - Reads the command line arguments. func getCommand(key: string): Option[CommandEnum] = case toLowerAscii(key) + of $ceIntrospect: + some(ceIntrospect) of $ceCreate: some(ceCreate) of $ceStart: @@ -139,16 +167,24 @@ proc getCommandLine*(): CommandLineArguments = else: case result.command of ceCreate: - if i == 1: - result.iso = some(key) - elif i == 2: - result.size = some(parseInt(key)) + if ".yaml" in key: + result.config = some(key) + i -= 1 + else: + if i == 1: + result.iso = some(key) + elif i == 2: + result.size = some(parseInt(key)) of ceStart: - if i == 1: - result.kernel = some(key) + if ".yaml" in key: + result.config = some(key) + i -= 1 else: - result.additionalStates &= key - of ceStop: + if i == 1: + result.kernel = some(key) + else: + result.additionalStates &= key + of ceStop, ceIntrospect: if i == 1: result.uuid = key of ceLs: @@ -174,8 +210,13 @@ proc getCommandLine*(): CommandLineArguments = of "kernel": if result.command == ceCreate: result.kernel = some(val) + of "shared": + result.shareddir = some(val) else: discard else: discard + if result.command in @[ceStart, ceCreate] and isNone(result.config): + echo("Config must be passed into arcd for these commands") + quit(1) proc getConfigFile*(args: CommandLineArguments): Config = ## getConfigFile - Loads a configuration file from disk into memory. @@ -231,6 +272,9 @@ proc getConfigFile*(args: CommandLineArguments): Config = if isSome(args.kernel): result.container.kernel = get(args.kernel) + if isSome(args.shareddir): + result.shareddir = args.shareddir + case args.command of ceCreate: if isSome(args.iso): diff --git a/src/libvfiopkg/types/connectivity.nim b/src/libvfio/types/connectivity.nim similarity index 100% rename from src/libvfiopkg/types/connectivity.nim rename to src/libvfio/types/connectivity.nim diff --git a/src/libvfiopkg/types/environment.nim b/src/libvfio/types/environment.nim similarity index 100% rename from src/libvfiopkg/types/environment.nim rename to src/libvfio/types/environment.nim diff --git a/src/libvfiopkg/types/hardware.nim b/src/libvfio/types/hardware.nim similarity index 96% rename from src/libvfiopkg/types/hardware.nim rename to src/libvfio/types/hardware.nim index d819bc9..9666ec3 100644 --- a/src/libvfiopkg/types/hardware.nim +++ b/src/libvfio/types/hardware.nim @@ -32,6 +32,10 @@ type vRam*: int ## vRAM available to the GPU. else: nil + Mdev* = object + uuid*: string ## UUID to access the mdev device + devId*: string ## Device name + const PassableVfios* = @[ ## List of PCI classes we can pass into the Arc ## containers. diff --git a/src/libvfiopkg/types/locks.nim b/src/libvfio/types/locks.nim similarity index 95% rename from src/libvfiopkg/types/locks.nim rename to src/libvfio/types/locks.nim index 0a337ae..b5f13f2 100644 --- a/src/libvfiopkg/types/locks.nim +++ b/src/libvfio/types/locks.nim @@ -11,6 +11,7 @@ type Lock* = object ## Lock object. config*: Config ## Hardware configuration. vfios*: seq[Vfio] ## List of VFIOs. + mdevs*: seq[Mdev] ## List of MDEV devices pidNum*: int ## PID number controlling the lock file. proc writeLockFile*(lockFile: string, lock: Lock) = diff --git a/src/libvfiopkg/types/process.nim b/src/libvfio/types/process.nim similarity index 100% rename from src/libvfiopkg/types/process.nim rename to src/libvfio/types/process.nim diff --git a/src/libvfiopkg/types/qmp.nim b/src/libvfio/types/qmp.nim similarity index 100% rename from src/libvfiopkg/types/qmp.nim rename to src/libvfio/types/qmp.nim diff --git a/src/libvfiopkg/utils/findPorts.nim b/src/libvfio/utils/findPorts.nim similarity index 100% rename from src/libvfiopkg/utils/findPorts.nim rename to src/libvfio/utils/findPorts.nim diff --git a/src/libvfiopkg/utils/getLocks.nim b/src/libvfio/utils/getLocks.nim similarity index 100% rename from src/libvfiopkg/utils/getLocks.nim rename to src/libvfio/utils/getLocks.nim diff --git a/src/libvfiopkg/utils/separateVfios.nim b/src/libvfio/utils/separateVfios.nim similarity index 100% rename from src/libvfiopkg/utils/separateVfios.nim rename to src/libvfio/utils/separateVfios.nim diff --git a/src/libvfiopkg/control/arguments.nim b/src/libvfiopkg/control/arguments.nim deleted file mode 100644 index 3986a92..0000000 --- a/src/libvfiopkg/control/arguments.nim +++ /dev/null @@ -1,221 +0,0 @@ -# -# Copyright: 2666680 Ontario Inc. -# Reason: Creates commands to execute on the host system. -# -import strformat -import strutils -import options -import os - -import ../types - -const - # Additional strings to help hide KVM status from the resources. - MachineConfig = join( - @["pc-q35-4.2", - "accel=kvm", - "usb=off", - "vmport=off", - "dump-guest-core=off" - ], - "," - ) - CpuConfig = join( - @["IvyBridge-IBRS", - "ss=on", - "vmx=on", - "pcid=on", - "hypervisor=on", - "arat=on", - "tsc-adjust=on", - "umip=on", - "md-clear=on", - "stibp=on", - "arch-capabilities=on", - "ssbd=on", - "xsaveopt=on", - "pdpe1gb=on", - "ibpb=on", - "ibrs=on", - "amd-stibp=on", - "amd-ssbd=on", - "skip-l1dfl-vmentry=on", - "pschange-mc-no=on", - "hv-vapic", - "hv-spinlocks=0x1fff", - "hv-vendor-id=1234567890ab", - "kvm=off" - ], - "," - ) - -proc runCommand*(command: Args): bool = - ## runCommand - Helper to run a command. - ## - ## Inputs - ## @command - Command to run. - ## - ## Returns - ## result - If the command was successful or not. - execShellCmd(join(@[command.exec] & command.args, " ")) == 0 - -func createKernel*(name: string, size: int): Args = - ## createKernel - Creates a kernel image for the Arc Container. - ## - ## Inputs - ## @name - Name of the kernel image. - ## @size - Initial size of the container. - ## - ## Returns - ## result - Arguments to create a kernel. - result.exec = "qemu-img" - result.args &= "create" - result.args &= "-f" - result.args &= "qcow2" - result.args &= name - result.args &= &"{size}G" - -func changeSocketGroup*(sudo: bool, sockets: seq[string]): Args = - ## changeSocketGroup - Changes the socket group to allow not to use sudo. - ## - ## Inputs - ## @sudo - Do we use sudo? - ## @sockets - Sockets to change. - ## - ## Returns - ## result - Arguments to change the socket group. - result.exec = "/usr/bin/chgrp" - - # If we need sudo - if sudo: - result.args &= result.exec - result.exec = "/bin/sudo" - - # Set the group to KVM - result.args &= "kvm" - - # Add all sockets - result.args &= sockets - -func changePermissions*(sudo: bool, sockets: seq[string]): Args = - ## changePermissions - Changes the permission on the socket. - ## - ## Inputs - ## @sudo - Do we use sudo? - ## @sockets - Sockets to change. - ## - ## Returns - ## result - Arguments to change the permissions for the sockets. - result.exec = "/usr/bin/chmod" - - # If we need sudo - if sudo: - result.args &= result.exec - result.exec = "/bin/sudo" - - # Set the group to KVM - result.args &= "g+rwx" - - # Add all sockets - result.args &= sockets - - -func qemuLaunch*(cfg: Config, uuid: string, - vfios: seq[Vfio], kernel: string, - install: bool, logDir: string, - sockets: seq[string]): Args = - ## qemuLaunch - Generates a qemu command to be executed. - ## - ## Inputs - ## @cfg - Configuration for the qemu launch. - ## @uuid - UUID of the app. - ## @vfios - List of VFIOs to put pass into the VM. - ## @kernel - Kernel path. - ## @install - Is this an installation? - ## @logDir - Directory for logging. - ## @sockets - Sockets to create. - ## - ## Returns - ## result - Arguments to launch a kernel. - func vfioArgs(device: Vfio): seq[string] = - if isGpu(device): - result &= "--device" - result &= &"vfio-pci,host={device},multifunction=on,display=off" - result &= "-mem-prealloc" - - let - qemuLogFile = logDir / (uuid & "-session.txt") - - # executing target - result.exec = "/bin/qemu-system-x86_64" - - # If we need sudo - if cfg.sudo: - result.args &= result.exec - result.exec = "/bin/sudo" - - # Log file - result.args &= "-D" - result.args &= qemuLogFile - - # UUID - Necessary for NVidia MDEV support - result.args &= "-uuid" - result.args &= uuid - - # Machine settings needed for NVidia MDEV support - result.args &= "-machine" - result.args &= MachineConfig - - # Cpu settings needed for NVidia MDEV support - result.args &= "-cpu" - result.args &= CpuConfig - - # clock settings - result.args &= "-rtc" - result.args &= "clock=host,base=localtime" - - # RAM allocation - result.args &= "-m" - result.args &= $cfg.cpus.ramAlloc - - # CPU configuration - result.args &= "-smp" - result.args &= $cfg.cpus - - # Kernel path - result.args &= "-hda" - result.args &= kernel - - # Enable KVM - result.args &= "--enable-kvm" - - # Set GPU vfio - for vfio in vfios: - result.args &= vfioArgs(vfio) - - # Enable sound - result.args &= "--soundhw" - result.args &= "all" - - # Enable QMP socket for all sockets. - for socket in sockets: - result.args &= "-qmp" - result.args &= &"unix:{socket},server,nowait" - - # States - for state in cfg.container.state: - result.args &= "-hdd" - result.args &= cfg.root / "states" / state - - # If it is being installed - if install and isSome(cfg.container.iso): - result.args &= "-cdrom" - result.args &= get(cfg.container.iso) - - # Additional commands sent into the qemu command - # NOTE: Start Process quotes these commands. - for command in cfg.commands: - let values = join(command.values, ",") - result.args &= command.arg - if values != "": - result.args &= values