13213 Want development driver for accessing AMD DF

Reviewed by: Patrick Mooney <pmooney@pfmooney.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

usr/src/cmd/Makefile

@@ -489,6 +489,7 @@ i386_SUBDIRS=		\
 	acpihpd		\
 	addbadsec	\
 	ahciem		\
+	amdzen		\
 	bhyve		\
 	bhyvectl	\
 	biosdev		\
@@ -499,7 +500,6 @@ i386_SUBDIRS=		\
 	rdmsr		\
 	rtc		\
 	ucodeadm	\
-	usmn		\
 	xhci		\
 	xvm
 

usr/src/cmd/usmn/Makefile → usr/src/cmd/amdzen/Makefile

@@ -13,7 +13,7 @@
 # Copyright 2020 Oxide Computer Company
 #
 
-PROG= usmn
+PROG= usmn udf
 
 include ../Makefile.cmd
 

usr/src/cmd/amdzen/udf.c

@@ -0,0 +1,118 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Oxide Computer Company
+ */
+
+/*
+ * Facilitate access to the AMD Zen data fabric
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <err.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <strings.h>
+#include <zen_udf.h>
+
+static void
+udf_readone(int fd, uint8_t inst, uint8_t func, uint16_t reg, boolean_t do64)
+{
+	int ret;
+	zen_udf_io_t zui;
+
+	bzero(&zui, sizeof (zui));
+	zui.zui_inst = inst;
+	zui.zui_func = func;
+	zui.zui_reg = reg;
+
+	ret = ioctl(fd, do64 ? ZEN_UDF_READ64 : ZEN_UDF_READ32, &zui);
+	if (ret != 0) {
+		err(EXIT_FAILURE, "failed to issue read ioctl");
+	}
+
+	(void) printf("ifr %x/%x/%x: 0x%" PRIx64 "\n",
+	    inst, func, reg, zui.zui_data);
+}
+
+int
+main(int argc, char *argv[])
+{
+	int c, fd;
+	const char *device = NULL;
+	const char *funcstr = NULL;
+	const char *inststr = NULL;
+	const char *regstr = NULL;
+	uint8_t func, inst;
+	uint16_t reg;
+	unsigned long lval;
+	char *eptr;
+	boolean_t do64 = B_FALSE;
+
+	while ((c = getopt(argc, argv, "d:f:i:r:l")) != -1) {
+		switch (c) {
+		case 'd':
+			device = optarg;
+			break;
+		case 'f':
+			funcstr = optarg;
+			break;
+		case 'i':
+			inststr = optarg;
+			break;
+		case 'l':
+			do64 = B_TRUE;
+			break;
+		case 'r':
+			regstr = optarg;
+			break;
+		}
+	}
+
+	if (device == NULL || funcstr == NULL || inststr == NULL ||
+	    regstr == NULL) {
+		warnx("missing required arguments");
+		(void) fprintf(stderr, "Usage: udf [-l] -d device -f func -i "
+		    "inst -r reg\n");
+	}
+
+	errno = 0;
+	lval = strtoul(funcstr, &eptr, 0);
+	if (errno != 0 || lval > UINT8_MAX || *eptr != '\0') {
+		errx(EXIT_FAILURE, "failed to parse -f: %s", funcstr);
+	}
+	func = (uint8_t)lval;
+
+	lval = strtoul(inststr, &eptr, 0);
+	if (errno != 0 || lval > UINT8_MAX || *eptr != '\0') {
+		errx(EXIT_FAILURE, "failed to parse -i: %s", inststr);
+	}
+	inst = (uint8_t)lval;
+
+	lval = strtoul(regstr, &eptr, 0);
+	if (errno != 0 || lval > UINT16_MAX || *eptr != '\0') {
+		errx(EXIT_FAILURE, "failed to parse -r: %s", regstr);
+	}
+	reg = (uint16_t)lval;
+
+	if ((fd = open(device, O_RDONLY)) < 0) {
+		err(EXIT_FAILURE, "failed to open %s", device);
+	}
+
+	udf_readone(fd, inst, func, reg, do64);
+	(void) close(fd);
+	return (0);
+}

usr/src/man/man7d/Makefile

@@ -245,6 +245,7 @@ i386_MANFILES=	ahci.7d		\
 		virtio.7d	\
 		wpi.7d		\
 		xhci.7d		\
+		zen_udf.7d	\
 		zyd.7d
 
 _MANLINKS=	1394.7d		\

usr/src/man/man7d/amdzen.7d

@@ -45,4 +45,6 @@ drivers are limited to
 .Sy x86
 platforms with AMD Family 17h processors.
 .Sh SEE ALSO
-.Xr smntemp 7D
+.Xr smntemp 7D ,
+.Xr usmn 7D ,
+.Xr zen_udf 7D

usr/src/man/man7d/usmn.7d

@@ -33,4 +33,5 @@ This driver is intended strictly for facilitating platform development
 and is not recommended for systems that aren't doing kernel development
 on AMD Zen platforms.
 .Sh SEE ALSO
-.Xr amdzen 7D
+.Xr amdzen 7D ,
+.Xr zen_udf 7D

usr/src/man/man7d/zen_udf.7d

@@ -0,0 +1,36 @@
+.\"
+.\" This file and its contents are supplied under the terms of the
+.\" Common Development and Distribution License ("CDDL"), version 1.0.
+.\" You may only use this file in accordance with the terms of version
+.\" 1.0 of the CDDL.
+.\"
+.\" A full copy of the text of the CDDL should have accompanied this
+.\" source.  A copy of the CDDL is also available via the Internet at
+.\" http://www.illumos.org/license/CDDL.
+.\"
+.\"
+.\" Copyright 2020 Oxide Computer Company
+.\"
+.Dd October 7, 2020
+.Dt ZEN_UDF 7D
+.Os
+.Sh NAME
+.Nm zen_udf
+.Nd AMD data fabric user access driver
+.Sh SYNOPSIS
+.Pa /devices/pseudo/amdzen@0/zen_udf@3:zen_udf.*
+.Sh DESCRIPTION
+The
+.Nm
+driver provides the ability to read data from the AMD data fabric
+.Pq DF
+on AMD Family 17h
+.Pq Zen, Zen+, and Zen 2
+processors.
+.Pp
+This driver is intended strictly for facilitating platform development
+and is not recommended for systems that aren't doing kernel development
+on AMD Zen platforms.
+.Sh SEE ALSO
+.Xr amdzen 7D ,
+.Xr usmn 7D

usr/src/pkg/manifests/driver-developer-amd-zen.mf

@@ -27,7 +27,11 @@ dir path=usr/lib
 dir path=usr/share/man
 dir path=usr/share/man/man7d
 driver name=usmn
+driver name=zen_udf
 file path=kernel/drv/$(ARCH64)/usmn group=sys
+file path=kernel/drv/$(ARCH64)/zen_udf group=sys
+file path=usr/lib/udf mode=0555
 file path=usr/lib/usmn mode=0555
 file path=usr/share/man/man7d/usmn.7d
+file path=usr/share/man/man7d/zen_udf.7d
 license lic_CDDL license=lic_CDDL

usr/src/uts/intel/Makefile.files

@@ -350,3 +350,4 @@ AMDZEN_OBJS = amdzen.o
 AMDZEN_STUB_OBJS = amdzen_stub.o
 SMNTEMP_OBJS = smntemp.o
 USMN_OBJS = usmn.o
+ZEN_UDF_OBJS = zen_udf.o

usr/src/uts/intel/Makefile.intel

@@ -776,4 +776,4 @@ DRV_KMODS	+= smntemp
 #
 DRV_KMODS	+= amdzen
 DRV_KMODS	+= amdzen_stub
-DRV_KMODS	+= usmn
+DRV_KMODS	+= usmn zen_udf

usr/src/uts/intel/io/amdzen/amdzen.c

@@ -157,7 +157,8 @@ typedef struct {
 
 static const amdzen_child_data_t amdzen_children[] = {
 	{ "smntemp", AMDZEN_C_SMNTEMP },
-	{ "usmn", AMDZEN_C_USMN }
+	{ "usmn", AMDZEN_C_USMN },
+	{ "zen_udf", AMDZEN_C_ZEN_UDF }
 };
 
 static uint32_t
@@ -166,6 +167,12 @@ amdzen_stub_get32(amdzen_stub_t *stub, off_t reg)
 	return (pci_config_get32(stub->azns_cfgspace, reg));
 }
 
+static uint64_t
+amdzen_stub_get64(amdzen_stub_t *stub, off_t reg)
+{
+	return (pci_config_get64(stub->azns_cfgspace, reg));
+}
+
 static void
 amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val)
 {
@@ -189,6 +196,24 @@ amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, uint8_t func,
 	return (amdzen_stub_get32(df->adf_funcs[4], AMDZEN_DF_F4_FICAD_LO));
 }
 
+/*
+ * Perform a targeted 64-bit indirect read to a specific instance and function.
+ */
+static uint64_t
+amdzen_df_read64(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, uint8_t func,
+    uint16_t reg)
+{
+	uint32_t val;
+
+	VERIFY(MUTEX_HELD(&azn->azn_mutex));
+	val = AMDZEN_DF_F4_FICAA_TARG_INST | AMDZEN_DF_F4_FICAA_SET_REG(reg) |
+	    AMDZEN_DF_F4_FICAA_SET_FUNC(func) |
+	    AMDZEN_DF_F4_FICAA_SET_INST(inst) | AMDZEN_DF_F4_FICAA_SET_64B;
+	amdzen_stub_put32(df->adf_funcs[4], AMDZEN_DF_F4_FICAA, val);
+	return (amdzen_stub_get64(df->adf_funcs[4], AMDZEN_DF_F4_FICAD_LO));
+}
+
+
 static uint32_t
 amdzen_smn_read32(amdzen_t *azn, amdzen_df_t *df, uint32_t reg)
 {
@@ -260,6 +285,45 @@ amdzen_c_df_count(void)
 	return (ret);
 }
 
+int
+amdzen_c_df_read32(uint_t dfno, uint8_t inst, uint8_t func,
+    uint16_t reg, uint32_t *valp)
+{
+	amdzen_df_t *df;
+	amdzen_t *azn = amdzen_data;
+
+	mutex_enter(&azn->azn_mutex);
+	df = amdzen_df_find(azn, dfno);
+	if (df == NULL) {
+		mutex_exit(&azn->azn_mutex);
+		return (ENOENT);
+	}
+
+	*valp = amdzen_df_read32(azn, df, inst, func, reg);
+	mutex_exit(&azn->azn_mutex);
+
+	return (0);
+}
+
+int
+amdzen_c_df_read64(uint_t dfno, uint8_t inst, uint8_t func,
+    uint16_t reg, uint64_t *valp)
+{
+	amdzen_df_t *df;
+	amdzen_t *azn = amdzen_data;
+
+	mutex_enter(&azn->azn_mutex);
+	df = amdzen_df_find(azn, dfno);
+	if (df == NULL) {
+		mutex_exit(&azn->azn_mutex);
+		return (ENOENT);
+	}
+
+	*valp = amdzen_df_read64(azn, df, inst, func, reg);
+	mutex_exit(&azn->azn_mutex);
+
+	return (0);
+}
 
 static boolean_t
 amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd)
@@ -269,14 +333,14 @@ amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd)
 
 	if (ndi_devi_alloc(azn->azn_dip, acd->acd_name,
 	    (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) {
-		dev_err(azn->azn_dip, CE_WARN, "failed to allocate child "
+		dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child "
 		    "dip for %s", acd->acd_name);
 		return (B_FALSE);
 	}
 
 	ddi_set_parent_data(child, (void *)acd);
 	if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) {
-		dev_err(azn->azn_dip, CE_WARN, "failed to online child "
+		dev_err(azn->azn_dip, CE_WARN, "!failed to online child "
 		    "dip %s: %d", acd->acd_name, ret);
 		return (B_FALSE);
 	}

usr/src/uts/intel/io/amdzen/amdzen.h

@@ -289,7 +289,8 @@ typedef struct amdzen {
 
 typedef enum {
 	AMDZEN_C_SMNTEMP = 1,
-	AMDZEN_C_USMN
+	AMDZEN_C_USMN,
+	AMDZEN_C_ZEN_UDF
 } amdzen_child_t;
 
 /*

usr/src/uts/intel/io/amdzen/amdzen_client.h

@@ -26,8 +26,10 @@
 extern "C" {
 #endif
 
-extern int amdzen_c_smn_read32(uint_t df, uint32_t reg, uint32_t *);
+extern int amdzen_c_smn_read32(uint_t, uint32_t, uint32_t *);
 extern uint_t amdzen_c_df_count(void);
+extern int amdzen_c_df_read32(uint_t, uint8_t, uint8_t, uint16_t, uint32_t *);
+extern int amdzen_c_df_read64(uint_t, uint8_t, uint8_t, uint16_t, uint64_t *);
 
 #ifdef __cplusplus
 }