This driver attaches to the Intel VMD drive and connects a new PCI do…

…main

starting at the max. domain, and then work down.  Then existing FreeBSD
drivers will attach.  Interrupt routing from the VMD MSI-X to the NVME
drive is not well known, so any interrupt is sent to all children that
register.

VROC used Intel meta data so graid(8) works with it. However, graid(8)
supports RAID 0,1,10 for read and write. I have some early code to
support writes with RAID 5.  Note that RAID 5 can have life issues
with SSDs since it can cause write amplification from updating the parity
data.

Hot plug support needs a change to skip the following check to work:
	if (pcib_request_feature(dev, PCI_FEATURE_HP) != 0) {
in sys/dev/pci/pci_pci.c.

Looked at by: imp, rpokala, bcr
Differential Revision:	https://reviews.freebsd.org/D21383

share/man/man4/Makefile

@@ -543,6 +543,7 @@ MAN=	aac.4 \
 	vkbd.4 \
 	vlan.4 \
 	vxlan.4 \
+	${_vmd.4} \
 	${_vmm.4} \
 	${_vmx.4} \
 	vpo.4 \
@@ -835,6 +836,7 @@ _qlxgbe.4=	qlxgbe.4
 _qlnxe.4=	qlnxe.4
 _sfxge.4=	sfxge.4
 _smartpqi.4=	smartpqi.4
+_vmd.4=		vmd.4
 
 MLINKS+=qlxge.4 if_qlxge.4
 MLINKS+=qlxgb.4 if_qlxgb.4

share/man/man4/vmd.4

@@ -0,0 +1,64 @@
+.\"-
+.\" SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+.\"
+.\" Copyright 2019 Cisco Systems, Inc.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd Octber 9, 2019
+.Dt VMD 4
+.Os
+.Sh NAME
+.Nm vmd
+.Nd Intel Volume Management Device driver
+.Sh SYNOPSIS
+To compile this driver into the kernel, place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset -indent
+.Cd "device vmd"
+.Cd "device vmd_bus"
+.Ed
+.Pp
+Alternatively, to load the driver as a module at boot time, place the following
+line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+vmd_load="YES"
+.Ed
+.Pp
+.Sh DESCRIPTION
+This driver attaches to Intel VMD devices as a new PCI domain and then
+triggers a probe of PCI devices.
+Intel VMD is used with Intel's VROC (Virtual RAID on chip) used with
+NVME drives on Skylake SP servers.
+.Sh SEE ALSO
+.Xr graid 8
+.Sh HISTORY
+The
+.Nm
+driver first appeared in
+.Fx 13.0 .
+.Sh BUGS
+.Nm
+is currently only available on amd64.

sys/amd64/conf/GENERIC

@@ -189,6 +189,10 @@ device		twe			# 3ware ATA RAID
 device		nvme			# base NVMe driver
 device		nvd			# expose NVMe namespaces as disks, depends on nvme
 
+# Intel Volume Management Device (VMD) support
+device		vmd			# base VMD device
+device		vmd_bus			# bus for VMD children
+
 # atkbdc0 controls both the keyboard and the PS/2 mouse
 device		atkbdc			# AT keyboard controller
 device		atkbd			# AT keyboard

sys/amd64/conf/NOTES

@@ -460,6 +460,11 @@ options 	ISCI_LOGGING	# enable debugging in isci HAL
 device         nvme    # base NVMe driver
 device         nvd     # expose NVMe namespaces as disks, depends on nvme
 
+#
+# Intel Volume Management Device (VMD) support
+device		vmd		# base VMD device
+device		vmd_bus		# bus for VMD children
+
 #
 # PMC-Sierra SAS/SATA controller
 device		pmspcv

sys/conf/files.amd64

@@ -226,6 +226,8 @@ dev/ntb/ntb_hw/ntb_hw_amd.c	optional	ntb_hw_amd | ntb_hw
 dev/ntb/ntb_hw/ntb_hw_intel.c	optional	ntb_hw_intel | ntb_hw
 dev/ntb/ntb_hw/ntb_hw_plx.c	optional	ntb_hw_plx | ntb_hw
 dev/ntb/test/ntb_tool.c		optional	ntb_tool
+dev/vmd/vmd.c			optional	vmd
+dev/vmd/vmd_bus.c		optional	vmd_bus
 dev/nvram/nvram.c		optional	nvram isa
 dev/random/ivy.c		optional	rdrand_rng !random_loadable
 dev/random/nehemiah.c		optional	padlock_rng !random_loadable