13689 Want AWS ENA driver

Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Approved by: Dan McDonald <danmcd@joyent.com>

usr/src/man/man7d/Makefile

@@ -16,6 +16,7 @@
 # Copyright 2016 Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
 # Copyright 2018 Nexenta Systems, Inc.
 # Copyright 2020 Peter Tribble
+# Copyright 2021 Oxide Computer Company
 #
 
 include		$(SRC)/Makefile.master
@@ -46,6 +47,7 @@ _MANFILES=	aac.7d		\
 		dtrace.7d	\
 		e1000g.7d	\
 		ehci.7d		\
+		ena.7d		\
 		fasttrap.7d	\
 		fbt.7d		\
 		fcip.7d		\

usr/src/man/man7d/ena.7d

@@ -0,0 +1,135 @@
+.\"
+.\" This file and its contents are supplied under the terms of the
+.\" Common Development and Distribution License ("CDDL"), version 1.0.
+.\" You may only use this file in accordance with the terms of version
+.\" 1.0 of the CDDL.
+.\"
+.\" A full copy of the text of the CDDL should have accompanied this
+.\" source.  A copy of the CDDL is also available via the Internet at
+.\" http://www.illumos.org/license/CDDL.
+.\"
+.\"
+.\" Copyright 2021 Oxide Computer Company
+.\"
+.Dd Nov 17, 2021
+.Dt ENA 7D
+.Os
+.Sh NAME
+.Nm ena
+.Nd Driver for the AWS Elastic Network Adapter
+.Sh SYNOPSIS
+.Pa /dev/net/ena*
+.Sh DESCRIPTION
+The
+.Sy ena
+driver is a GLDv3 NIC driver for the AWS Elastic Network Adapter
+family of virtual devices.
+The driver supports:
+.Bl -dash -offset indent
+.It
+Jumbo frames up to 9216 bytes.
+.It
+Multiple Rx and Tx rings.
+.El
+.Pp
+By design, this driver does not support VNICs.
+A given ENA device can only ever receive traffic for a single unicast
+MAC address and IP address combination, as determined by the AWS configuration.
+There is no support for promiscuous mode, or for receiving traffic for
+additional unicast or multicast addresses.
+.Sh CONFIGURATION
+The
+.Sy ena.conf
+file contains user configurable parameters, each of which is described
+below.
+This file is read when an ENA device is found and an instance of the
+driver is attached to it.
+Changes made to this file do not affect running instances.
+Only instances attached after the changes will see the effects of
+those changes.
+Therefore, if you want your change to take effect on a running
+instance, you must somehow reload it.
+That could be done by a manual reloading of the driver or a system
+reboot.
+.Sh PROPERTIES
+The configuration file can be found at
+.Pa /kernel/drv/ena.conf .
+.Bl -hang -width Ds
+.It Sy rx_queue_num_descs
+.Bd -filled -compact
+Minimum:
+.Sy 64 |
+Maximum:
+.Sy device dependent
+.Ed
+.Bd -filled -compact
+Default:
+.Sy device maximum
+.Ed
+.Bd -filled
+The
+.Sy rx_queue_num_descs
+property determines the number of descriptors provided by the Rx queue.
+Currently a single descriptor is equal to a single packet, but in the
+future it may be that a single packet consumes multiple descriptors.
+.Ed
+.It Sy rx_queue_intr_limit
+.Bd -filled -compact
+Minimum:
+.Sy 16 |
+Maximum:
+.Sy 4096
+.Ed
+.Bd -filled -compact
+Default:
+.Sy 256
+.Ed
+.Bd -filled
+The
+.Sy rx_queue_intr_limit
+property determines the number frames an Rx interrupt will attempt to
+process before returning and claiming the interrupt.
+This is meant to keep the ENA Rx interrupt handler from consuming too
+much system time.
+In general, when a NIC becomes saturated with packets, the
+.Sy MAC
+layer will switch the driver into polling mode to reduce interrupt
+load.
+.Ed
+.It Sy tx_queue_num_descs
+.Bd -filled -compact
+Minimum:
+.Sy 64 |
+Maximum:
+.Sy device dependent
+.Ed
+.Bd -filled -compact
+Default:
+.Sy device maximum
+.Ed
+.Bd -filled
+The
+.Sy tx_queue_num_descs
+property determines the number of descriptors provided by the Tx queue.
+Currently a single descriptor is equal to a single packet, but in the
+future it may be that a single packet consumes multiple descriptors.
+.Ed
+.El
+.Sh FILES
+.Bl -tag -width Pa
+.It Pa /kernel/drv/amd64/ena
+Device driver (x86)
+.It Pa /kernel/drv/ena.conf
+Driver configuration file containing user-configurable options
+.El
+.Sh INTERFACE STABILITY
+The tunables in
+.Pa ena.conf
+are considered
+.Sy Evolving
+and may change in the future.
+.Sh SEE ALSO
+.Xr dladm 1M ,
+.Xr snoop 1M ,
+.Xr driver.conf 4 ,
+.Xr dlpi 7P

usr/src/pkg/manifests/driver-network-ena.p5m

@@ -0,0 +1,36 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2021 Oxide Computer Company
+#
+
+<include global_zone_only_component>
+set name=pkg.fmri value=pkg:/driver/network/ena@$(PKGVERS)
+set name=pkg.summary value="AWS ENA Ethernet Driver"
+set name=pkg.description value="AWS ENA Ethernet Driver"
+set name=info.classification \
+    value=org.opensolaris.category.2008:Drivers/Networking
+set name=variant.arch value=i386
+dir  path=kernel group=sys
+dir  path=kernel/drv group=sys
+dir  path=kernel/drv/$(ARCH64) group=sys
+file path=kernel/drv/$(ARCH64)/ena group=sys
+file path=kernel/drv/ena.conf group=sys
+dir  path=usr/share/man
+dir  path=usr/share/man/man7d
+file path=usr/share/man/man7d/ena.7d
+driver name=ena perms="* 0666 root sys" clone_perms="ena 0666 root sys" \
+    alias=pciex1d0f,ec2 \
+    alias=pciex1d0f,1ec2 \
+    alias=pciex1d0f,ec20 \
+    alias=pciex1d0f,ec21
+license lic_CDDL license=lic_CDDL

usr/src/uts/common/Makefile.files

@@ -29,6 +29,7 @@
 # Copyright 2016 OmniTI Computer Consulting, Inc.  All rights reserved.
 # Copyright 2016 Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
 # Copyright 2020 RackTop Systems, Inc.
+# Copyright 2021 Oxide Computer Company
 #
 
 #
@@ -2288,3 +2289,10 @@ BNX_OBJS += \
 #
 MLXCX_OBJS +=	mlxcx.o mlxcx_dma.o mlxcx_cmd.o mlxcx_intr.o mlxcx_gld.o \
 		mlxcx_ring.o mlxcx_sensor.o
+
+#
+# ena(7D)
+#
+ENA_OBJS +=	ena.o ena_admin.o ena_dma.o ena_gld.o ena_hw.o ena_intr.o \
+		ena_stats.o ena_tx.o ena_rx.o
+

usr/src/uts/common/Makefile.rules

@@ -26,7 +26,7 @@
 # Copyright 2019 Joyent, Inc.
 # Copyright 2018 Nexenta Systems, Inc.
 # Copyright (c) 2017 by Delphix. All rights reserved.
-# Copyright 2020 Oxide Computer Company
+# Copyright 2021 Oxide Computer Company
 #
 
 #
@@ -777,6 +777,10 @@ $(OBJS_DIR)/%.o:		$(UTSBASE)/common/io/elxl/%.c
 	$(COMPILE.c) -o $@ $<
 	$(CTFCONVERT_O)
 
+$(OBJS_DIR)/%.o:		$(UTSBASE)/common/io/ena/%.c
+	$(COMPILE.c) -o $@ $<
+	$(CTFCONVERT_O)
+
 $(OBJS_DIR)/%.o:		$(UTSBASE)/common/io/fcoe/%.c
 	$(COMPILE.c) -o $@ $<
 	$(CTFCONVERT_O)