Skip to content

Commit

Permalink
hw/npu2, platform: Add NPU2 platform device detection callback
Browse files Browse the repository at this point in the history 
There is no standardised way to determine the presence and type of devices
connected to an NPU on POWER9.

Currently, we hardcode device types based on platform type (as no platform
currently supports both OpenCAPI and NVLink), and for OpenCAPI platforms
we use I2C to detect presence.

Witherspoon (and potentially other platforms later on) supports both
NVLink and OpenCAPI, and additionally uses SXM2 connectors which can carry
more than one link, rather than the SlimSAS connectors used for OpenCAPI on
Zaius and ZZ. This necessitates some special handling.

Add a platform callback for NPU device detection. In a later patch, we
will use this to implement Witherspoon-specific device detection. For now,
add a Witherspoon stub that sets all links to NVLink (i.e. current
behaviour).

Move the existing I2C-based presence detection for OpenCAPI devices on
Zaius/ZZ into common code, which we use by default for platforms which do
not define a callback. Clean up the use of the ibm,npu-link-type property,
which will now be exposed solely for debugging and not consumed internally.

Signed-off-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>
Reviewed-by: Frederic Barrat <fbarrat@linux.ibm.com>
Signed-off-by: Stewart Smith <stewart@linux.ibm.com>
  • Loading branch information
@ajdlinux @stewartsmith
ajdlinux authored and stewartsmith committed Sep 18, 2018
1 parent 68415d5 commit b6cc82c
Show file tree
Hide file tree
Showing 9 changed files with 144 additions and 97 deletions.
2 changes: 2 additions & 0 deletions core/platform.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <errorlog.h>
#include <bt.h>
#include <nvram.h>
#include <npu2.h>
#include <platforms/astbmc/astbmc.h>

bool manufacturing_mode = false;
Expand Down Expand Up @@ -204,6 +205,7 @@ static struct platform generic_platform = {
.start_preload_resource = generic_start_preload_resource,
.resource_loaded = generic_resource_loaded,
.ocapi = &generic_ocapi,
.npu2_device_detect = npu2_i2c_presence_detect, /* Assumes ZZ */
};

const struct bmc_platform *bmc_platform = &generic_bmc;
Expand Down
1 change: 0 additions & 1 deletion hdata/spira.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1495,7 +1495,6 @@ static void add_npu(struct dt_node *xscom, const struct HDIF_array_hdr *links,
}

dt_add_property_string(node, "compatible", "ibm,npu-link");
dt_add_property_string(node, "ibm,npu-link-type", "nvlink");
dt_add_property_cells(node, "reg", link_count);
dt_add_property_cells(node, "ibm,npu-link-index", link_count);
dt_add_property_cells(node, "ibm,workbook-link-id", link_id);
Expand Down
112 changes: 98 additions & 14 deletions hw/npu2-common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,7 @@
#include <npu2-regs.h>
#include <bitutils.h>
#include <nvram.h>

enum npu2_dev_type npu2_dt_link_dev_type(struct dt_node *link)
{
const char *link_type = dt_prop_get(link, "ibm,npu-link-type") ?:
"unknown";
if (streq(link_type, "nvlink")) {
return NPU2_DEV_TYPE_NVLINK;
} else if (streq(link_type, "opencapi")) {
return NPU2_DEV_TYPE_OPENCAPI;
} else {
return NPU2_DEV_TYPE_UNKNOWN;
}
}
#include <i2c.h>

/*
* We use the indirect method because it uses the same addresses as
Expand Down Expand Up @@ -109,12 +97,62 @@ void npu2_write_mask_4b(struct npu2 *p, uint64_t reg, uint32_t val, uint32_t mas
(uint64_t)new_val << 32);
}

static bool _i2c_presence_detect(struct npu2_dev *dev)
{
uint8_t state, data;
int rc;

rc = i2c_request_send(dev->npu->i2c_port_id_ocapi,
platform.ocapi->i2c_presence_addr,
SMBUS_READ, 0, 1,
&state, 1, 120);
if (rc) {
OCAPIERR(dev, "error detecting link presence: %d\n", rc);
return true; /* assume link exists */
}

OCAPIDBG(dev, "I2C presence detect: 0x%x\n", state);

switch (dev->link_index) {
case 2:
data = platform.ocapi->i2c_presence_odl0;
break;
case 3:
data = platform.ocapi->i2c_presence_odl1;
break;
default:
OCAPIERR(dev, "presence detection on invalid link\n");
return true;
}
/* Presence detect bits are active low */
return !(state & data);
}

/*
* A default presence detection implementation for platforms like ZZ and Zaius
* that don't implement their own. Assumes all devices found will be OpenCAPI.
*/
void npu2_i2c_presence_detect(struct npu2 *npu)
{
struct npu2_dev *dev;
assert(platform.ocapi);
for (int i = 0; i < npu->total_devices; i++) {
dev = &npu->devices[i];
if (platform.ocapi->force_presence ||
_i2c_presence_detect(dev))
dev->type = NPU2_DEV_TYPE_OPENCAPI;
else
dev->type = NPU2_DEV_TYPE_UNKNOWN;
}
}

static struct npu2 *setup_npu(struct dt_node *dn)
{
struct npu2 *npu;
struct npu2_dev *dev;
struct dt_node *np;
uint32_t num_links;
char port_name[17];
void *npumem;
char *path;
int gcid;
Expand All @@ -138,6 +176,28 @@ static struct npu2 *setup_npu(struct dt_node *dn)
npu->chip_id = gcid;
npu->xscom_base = dt_get_address(dn, 0, NULL);
npu->phb_index = dt_prop_get_u32(dn, "ibm,phb-index");

if (platform.ocapi) {
/* Find I2C port for handling device presence/reset */
snprintf(port_name, sizeof(port_name), "p8_%08x_e%dp%d",
gcid, platform.ocapi->i2c_engine,
platform.ocapi->i2c_port);
prlog(PR_DEBUG, "NPU: Looking for I2C port %s\n", port_name);

dt_for_each_compatible(dt_root, np, "ibm,power9-i2c-port") {
if (streq(port_name, dt_prop_get(np, "ibm,port-name"))) {
npu->i2c_port_id_ocapi = dt_prop_get_u32(np, "ibm,opal-id");
break;
}
}

if (!npu->i2c_port_id_ocapi) {
prlog(PR_ERR, "NPU: Couldn't find I2C port %s\n",
port_name);
goto failed;
}
}

npu->devices = npumem + sizeof(struct npu2);

dt_for_each_compatible(dn, np, "ibm,npu-link") {
Expand All @@ -146,7 +206,8 @@ static struct npu2 *setup_npu(struct dt_node *dn)
dev->link_index = dt_prop_get_u32(np, "ibm,npu-link-index");
/* May be overridden by platform presence detection */
dev->brick_index = dev->link_index;
dev->type = npu2_dt_link_dev_type(np);
/* Will be overridden by presence detection */
dev->type = NPU2_DEV_TYPE_UNKNOWN;
dev->npu = npu;
dev->dt_node = np;
dev->pl_xscom_base = dt_prop_get_u64(np, "ibm,npu-phy");
Expand All @@ -161,6 +222,12 @@ static struct npu2 *setup_npu(struct dt_node *dn)
prlog(PR_INFO, " SCOM Base: %08llx\n", npu->xscom_base);
free(path);
return npu;

failed:
prlog(PR_ERR, "NPU: Chip %d NPU setup failed\n", gcid);
free(path);
free(npu);
return NULL;
}

static void setup_devices(struct npu2 *npu)
Expand All @@ -177,13 +244,22 @@ static void setup_devices(struct npu2 *npu)
switch (dev->type) {
case NPU2_DEV_TYPE_NVLINK:
nvlink_detected = true;
dt_add_property_strings(dev->dt_node,
"ibm,npu-link-type",
"nvlink");
break;
case NPU2_DEV_TYPE_OPENCAPI:
ocapi_detected = true;
dt_add_property_strings(dev->dt_node,
"ibm,npu-link-type",
"opencapi");
break;
default:
prlog(PR_INFO, "NPU: Link %d device not present\n",
npu->devices[i].link_index);
dt_add_property_strings(dev->dt_node,
"ibm,npu-link-type",
"unknown");
}
}

Expand Down Expand Up @@ -221,8 +297,16 @@ void probe_npu2(void)
prlog(PR_WARNING, "NPU2: Using ZCAL impedance override = %d\n", nv_zcal_nominal);
}

if (!platform.npu2_device_detect) {
prlog(PR_INFO, "NPU: Platform does not support NPU\n");
return;
}

dt_for_each_compatible(dt_root, np, "ibm,power9-npu") {
npu = setup_npu(np);
if (!npu)
continue;
platform.npu2_device_detect(npu);
setup_devices(npu);
}
}
93 changes: 14 additions & 79 deletions hw/npu2-opencapi.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,6 @@
#include <i2c.h>
#include <nvram.h>

#define OCAPIDBG(dev, fmt, a...) prlog(PR_DEBUG, "OCAPI[%d:%d]: " fmt, \
dev->npu->chip_id, dev->brick_index, ## a)
#define OCAPIINF(dev, fmt, a...) prlog(PR_INFO, "OCAPI[%d:%d]: " fmt, \
dev->npu->chip_id, dev->brick_index, ## a)
#define OCAPIERR(dev, fmt, a...) prlog(PR_ERR, "OCAPI[%d:%d]: " fmt, \
dev->npu->chip_id, dev->brick_index, ## a)


#define NPU_IRQ_LEVELS 35
#define NPU_IRQ_LEVELS_XSL 23
#define MAX_PE_HANDLE ((1 << 15) - 1)
Expand Down Expand Up @@ -842,7 +834,7 @@ static void assert_reset(struct npu2_dev *dev)
* register and a pin is in output mode if its value is 0
*/
data = ~pin;
rc = i2c_request_send(dev->i2c_port_id_ocapi,
rc = i2c_request_send(dev->npu->i2c_port_id_ocapi,
platform.ocapi->i2c_reset_addr, SMBUS_WRITE,
0x3, 1,
&data, sizeof(data), 120);
Expand All @@ -851,7 +843,7 @@ static void assert_reset(struct npu2_dev *dev)

/* register 1 controls the signal, reset is active low */
data = ~pin;
rc = i2c_request_send(dev->i2c_port_id_ocapi,
rc = i2c_request_send(dev->npu->i2c_port_id_ocapi,
platform.ocapi->i2c_reset_addr, SMBUS_WRITE,
0x1, 1,
&data, sizeof(data), 120);
Expand All @@ -874,7 +866,7 @@ static void deassert_reset(struct npu2_dev *dev)
int rc;

data = 0xFF;
rc = i2c_request_send(dev->i2c_port_id_ocapi,
rc = i2c_request_send(dev->npu->i2c_port_id_ocapi,
platform.ocapi->i2c_reset_addr, SMBUS_WRITE,
0x1, 1,
&data, sizeof(data), 120);
Expand All @@ -888,43 +880,6 @@ static void deassert_reset(struct npu2_dev *dev)
}
}

static bool i2c_presence_detect(struct npu2_dev *dev)
{
uint8_t state, data;
int rc;

/*
* Opencapi presence detection is done through i2c
*
* Lagrange platforms (ZZ, Zaius) use the same default mechanism.
* Witherspoon will need a specific implementation, TBD.
*/
rc = i2c_request_send(dev->i2c_port_id_ocapi,
platform.ocapi->i2c_presence_addr,
SMBUS_READ, 0, 1,
&state, 1, 120);
if (rc) {
OCAPIERR(dev, "error detecting link presence: %d\n", rc);
return true; /* assume link exists */
}

OCAPIDBG(dev, "I2C presence detect: 0x%x\n", state);

switch (dev->brick_index) { // TODO(ajd): Link or brick index?
case 2:
data = platform.ocapi->i2c_presence_odl0;
break;
case 3:
data = platform.ocapi->i2c_presence_odl1;
break;
default:
OCAPIERR(dev, "presence detection on invalid link\n");
return true;
}
/* Presence detect bits are active low */
return !(state & data);
}

static void reset_odl(uint32_t gcid, struct npu2_dev *dev)
{
uint64_t reg, config_xscom;
Expand Down Expand Up @@ -1015,17 +970,18 @@ static void start_training(uint32_t gcid, struct npu2_dev *dev)
xscom_write(gcid, config_xscom, reg);
}

static int64_t npu2_opencapi_get_presence_state(struct pci_slot *slot,
static int64_t npu2_opencapi_get_presence_state(struct pci_slot __unused *slot,
uint8_t *val)
{
bool present;
struct npu2_dev *dev = phb_to_npu2_dev_ocapi(slot->phb);

if (platform.ocapi->force_presence)
present = true;
else
present = i2c_presence_detect(dev);
*val = present;
/*
* Presence detection for OpenCAPI is currently done at the start of
* NPU initialisation, and we only create slots if a device is present.
* As such we will never be asked to get the presence of a slot that's
* empty.
*
* This may change if we ever support hotplug down the track.
*/
*val = true;
return OPAL_SUCCESS;
}

Expand Down Expand Up @@ -1578,9 +1534,8 @@ static void setup_debug_training_state(struct npu2_dev *dev)

static void setup_device(struct npu2_dev *dev)
{
struct dt_node *dn_phb, *dn;
struct dt_node *dn_phb;
struct pci_slot *slot;
char port_name[17];
uint64_t mm_win[2];

/* Populate PHB device node */
Expand Down Expand Up @@ -1626,23 +1581,6 @@ static void setup_device(struct npu2_dev *dev)
dev->bdfn = 0;
dev->train_need_fence = false;
dev->train_fenced = false;
/* Find I2C port for handling device reset */
snprintf(port_name, sizeof(port_name), "p8_%08x_e%dp%d",
dev->npu->chip_id, platform.ocapi->i2c_engine,
platform.ocapi->i2c_port);
prlog(PR_DEBUG, "OCAPI: Looking for I2C port %s\n", port_name);

dt_for_each_compatible(dt_root, dn, "ibm,power9-i2c-port") {
if (streq(port_name, dt_prop_get(dn, "ibm,port-name"))) {
dev->i2c_port_id_ocapi = dt_prop_get_u32(dn, "ibm,opal-id");
break;
}
}

if (!dev->i2c_port_id_ocapi) {
prlog(PR_ERR, "OCAPI: Couldn't find I2C port %s\n", port_name);
goto failed;
}

/* TODO: Procedure 13.1.3.7 - AFU Memory Range BARs */
/* Procedure 13.1.3.8 - AFU MMIO Range BARs */
Expand All @@ -1667,9 +1605,6 @@ static void setup_device(struct npu2_dev *dev)
}
pci_register_phb(&dev->phb_ocapi, OPAL_DYNAMIC_PHB_ID);
return;
failed:
dt_add_property_string(dn_phb, "status", "error");
return;
}

static void read_nvram_training_state(void)
Expand Down

0 comments on commit b6cc82c

Please sign in to comment.