virtlet-ds.yaml

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: virtlet
  namespace: kube-system
spec:
  selector:
    matchLabels:
      runtime: virtlet
  template:
    metadata:
      name: virtlet
      labels:
        runtime: virtlet
    spec:
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      # hostPID is true to (1) enable VMs to survive virtlet container restart
      # (to be checked) and (2) to enable the use of nsenter in init container
      hostPID: true
      # bootstrap procedure needs to create a configmap in kube-system namespace
      serviceAccountName: virtlet

      # only run Virtlet pods on the nodes with extraRuntime=virtlet label
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: extraRuntime
                operator: In
                values:
                - virtlet

      initContainers:
      # The init container copies virtlet's flexvolume driver
      # to the default kubelet plugin dir and ensures that
      # the directories needed by libvirt & virtlet exist on the host
      - name: prepare-node
        image: mirantis/virtlet
        imagePullPolicy: IfNotPresent
        command:
        - /prepare-node.sh
        volumeMounts:
        - name: k8s-flexvolume-plugins-dir
          mountPath: /kubelet-volume-plugins
        - name: run
          # Don't add "mountPropagation: Bidirectional", it will lack mount entry
          mountPath: /run
        - name: dockersock
          mountPath: /var/run/docker.sock
        - name: log
          mountPath: /hostlog
        # for ensuring that /var/lib/libvirt/images exists on node
        - name: var-lib
          mountPath: /host-var-lib
        - name: dev
          mountPath: /dev
        - mountPath: /var/lib/virtlet
          name: virtlet
        securityContext:
          privileged: true
        env:
        - name: KUBE_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: VIRTLET_DISABLE_KVM
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: disable_kvm
              optional: true
        - name: VIRTLET_SRIOV_SUPPORT
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: sriov_support
              optional: true
        - name: VIRTLET_DOWNLOAD_PROTOCOL
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: download_protocol
              optional: true
        - name: VIRTLET_LOGLEVEL
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: loglevel
              optional: true
        - name: VIRTLET_CALICO_SUBNET
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: calico-subnet
              optional: true
        - name: IMAGE_REGEXP_TRANSLATION
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: image_regexp_translation
              optional: true
        - name: VIRTLET_RAW_DEVICES
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: raw_devices
              optional: true
        - name: VIRTLET_DISABLE_LOGGING
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: disable_logging
              optional: true
        - name: VIRTLET_CPU_MODEL
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: cpu-model
              optional: true
        - name: KUBELET_ROOT_DIR
          valueFrom:
            configMapKeyRef:
              name: virtlet-config
              key: kubelet_root_dir
              optional: true
        - name: VIRTLET_IMAGE_TRANSLATIONS_DIR
          value: /etc/virtlet/images

      containers:
      - name: libvirt
        image: mirantis/virtlet
        # In case we inject local virtlet image we want to use it not officially available one
        imagePullPolicy: IfNotPresent
        command:
        - /libvirt.sh
        volumeMounts:
        - mountPath: /etc/libvirt/qemu
          name: qemu
        - mountPath: /sys/fs/cgroup
          name: cgroup
        - mountPath: /lib/modules
          name: modules
          readOnly: true
        - mountPath: /boot
          name: boot
          readOnly: true
        - mountPath: /run
          # Don't add "mountPropagation: Bidirectional", it will lack mount entry
          name: run
        - mountPath: /var/lib/virtlet
          name: virtlet
        - mountPath: /var/lib/libvirt
          name: libvirt
        - mountPath: /var/run/libvirt
          name: libvirt-sockets
        # the log dir is needed here because otherwise libvirt will produce errors
        # like this:
        # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory
        - name: vms-log
          mountPath: /var/log/vms
        - name: libvirt-log
          mountPath: /var/log/libvirt
        - name: dev
          mountPath: /dev
        securityContext:
          privileged: true
        readinessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null
      - name: virtlet
        image: mirantis/virtlet
        # In case we inject local virtlet image we want to use it not officially available one
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /etc/libvirt/qemu
          name: qemu
        - mountPath: /run
          # Don't add "mountPropagation: Bidirectional", it will lack mount entry
          name: run
        # /boot and /lib/modules are required by supermin
        - mountPath: /lib/modules
          name: modules
          readOnly: true
        - mountPath: /boot
          name: boot
          readOnly: true
        - name: dev
          mountPath: /dev
        - mountPath: /var/lib/virtlet
          name: virtlet
          mountPropagation: Bidirectional
        - mountPath: /var/lib/libvirt
          name: libvirt
        - mountPath: /var/run/libvirt
          name: libvirt-sockets
        - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
          name: k8s-flexvolume-plugins-dir
        - mountPath: /var/lib/kubelet/pods
          name: k8s-pods-dir
          mountPropagation: Bidirectional
        - name: vms-log
          mountPath: /var/log/vms
        - mountPath: /etc/virtlet/images
          name: image-name-translations
        - name: pods-log
          mountPath: /var/log/pods
        # needed for diagnostic purposes
        - name: libvirt-log
          mountPath: /var/log/libvirt
        - name: netns-dir
          mountPath: /var/run/netns
          mountPropagation: Bidirectional
        - name: cgroup
          mountPath: /sys/fs/cgroup
        securityContext:
          privileged: true
        readinessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - socat - UNIX:/run/virtlet.sock </dev/null
      - name: vms
        image: mirantis/virtlet
        imagePullPolicy: IfNotPresent
        command:
        - /vms.sh
        volumeMounts:
        - mountPath: /var/lib/virtlet
          name: virtlet
          mountPropagation: HostToContainer
        - mountPath: /var/lib/libvirt
          name: libvirt
        - name: vms-log
          mountPath: /var/log/vms
        - mountPath: /var/lib/kubelet/pods
          name: k8s-pods-dir
          mountPropagation: HostToContainer
        - name: dev
          mountPath: /dev
        - name: modules
          mountPath: /lib/modules
      volumes:
      # /dev is needed for host raw device access
      - hostPath:
          path: /dev
        name: dev
      - hostPath:
          path: /sys/fs/cgroup
        name: cgroup
      - hostPath:
          path: /lib/modules
        name: modules
      - hostPath:
          path: /boot
        name: boot
      - hostPath:
          path: /run
        name: run
      # TODO: don't hardcode docker socket location here
      # This will require CRI proxy installation to run
      # in host mount namespace.
      - hostPath:
          path: /var/run/docker.sock
        name: dockersock
      - hostPath:
          path: /var/lib/virtlet
        name: virtlet
      - hostPath:
          path: /var/lib/libvirt
        name: libvirt
      - hostPath:
          path: /var/log
        name: log
      - hostPath:
          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
        name: k8s-flexvolume-plugins-dir
      - hostPath:
          path: /var/lib/kubelet/pods
        name: k8s-pods-dir
      - hostPath:
          path: /var/lib
        name: var-lib
      - hostPath:
          path: /var/log/virtlet/vms
        name: vms-log
      - hostPath:
          path: /var/log/libvirt
        name: libvirt-log
      - hostPath:
          path: /var/run/libvirt
        name: libvirt-sockets
      - hostPath:
          path: /var/log/pods
        name: pods-log
      - hostPath:
          path: /var/run/netns
        name: netns-dir
      - hostPath:
          path: /etc/libvirt/qemu
        name: qemu
      - configMap:
          name: virtlet-image-translations
        name: image-name-translations
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: virtlet
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: virtlet
subjects:
- kind: ServiceAccount
  name: virtlet
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: virtlet
  namespace: kube-system
rules:
  - apiGroups:
    - ""
    resources:
    - configmaps
    - nodes
    verbs:
    - create
    - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: configmap-reader
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: virtlet-userdata-reader
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - secrets
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: kubelet-node-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: configmap-reader
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: vm-userdata-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: virtlet-userdata-reader
subjects:
- kind: ServiceAccount
  name: virtlet
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: virtlet-crd
rules:
- apiGroups:
  - "apiextensions.k8s.io"
  resources:
  - customresourcedefinitions
  verbs:
  - create
- apiGroups:
  - "virtlet.k8s"
  resources:
  - virtletimagemappings
  - virtletconfigmappings
  verbs:
  - list
  - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: virtlet-crd
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: virtlet-crd
subjects:
- kind: ServiceAccount
  name: virtlet
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: virtlet
  namespace: kube-system