Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
spapr: Remove support for NVIDIA V100 GPU with NVLink2 
NVLink2 support was removed from the PPC PowerNV platform and VFIO in
Linux 5.13 with commits :

  562d1e207d32 ("powerpc/powernv: remove the nvlink support")
  b392a1989170 ("vfio/pci: remove vfio_pci_nvlink2")

This was 2.5 years ago. Do the same in QEMU with a revert of commit
ec132ef ("spapr: Support NVIDIA V100 GPU with NVLink2"). Some
adjustements are required on the NUMA part.

Cc: Alexey Kardashevskiy <aik@ozlabs.ru>
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Signed-off-by: Cédric Le Goater <clg@redhat.com>
Message-ID: <20230918091717.149950-1-clg@kaod.org>
Signed-off-by: Daniel Henrique Barboza <danielhb413@gmail.com>
  • Loading branch information
@legoater @danielhb
legoater authored and danielhb committed Sep 18, 2023
1 parent 527b238 commit 44fa20c
Show file tree
Hide file tree
Showing 11 changed files with 14 additions and 708 deletions.
1 change: 0 additions & 1 deletion hw/ppc/meson.build
View file
Expand Up @@ -36,7 +36,6 @@ ppc_ss.add(when: ['CONFIG_PSERIES', 'CONFIG_TCG'], if_true: files(
ppc_ss.add(when: 'CONFIG_SPAPR_RNG', if_true: files('spapr_rng.c'))
ppc_ss.add(when: ['CONFIG_PSERIES', 'CONFIG_LINUX'], if_true: files(
'spapr_pci_vfio.c',
'spapr_pci_nvlink2.c'
))

# IBM PowerNV
Expand Down
22 changes: 4 additions & 18 deletions hw/ppc/spapr.c
View file
Expand Up @@ -2875,8 +2875,6 @@ static void spapr_machine_init(MachineState *machine)
/* init CPUs */
spapr_init_cpus(spapr);

spapr->gpu_numa_id = spapr_numa_initial_nvgpu_numa_id(machine);

/* Init numa_assoc_array */
spapr_numa_associativity_init(spapr, machine);

Expand Down Expand Up @@ -4134,7 +4132,6 @@ static bool spapr_phb_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
&sphb->buid, &sphb->io_win_addr,
&sphb->mem_win_addr, &sphb->mem64_win_addr,
windows_supported, sphb->dma_liobn,
&sphb->nv2_gpa_win_addr, &sphb->nv2_atsd_win_addr,
errp);
}

Expand Down Expand Up @@ -4397,8 +4394,7 @@ static const CPUArchIdList *spapr_possible_cpu_arch_ids(MachineState *machine)
static bool spapr_phb_placement(SpaprMachineState *spapr, uint32_t index,
uint64_t *buid, hwaddr *pio,
hwaddr *mmio32, hwaddr *mmio64,
unsigned n_dma, uint32_t *liobns,
hwaddr *nv2gpa, hwaddr *nv2atsd, Error **errp)
unsigned n_dma, uint32_t *liobns, Error **errp)
{
/*
* New-style PHB window placement.
Expand Down Expand Up @@ -4443,9 +4439,6 @@ static bool spapr_phb_placement(SpaprMachineState *spapr, uint32_t index,
*pio = SPAPR_PCI_BASE + index * SPAPR_PCI_IO_WIN_SIZE;
*mmio32 = SPAPR_PCI_BASE + (index + 1) * SPAPR_PCI_MEM32_WIN_SIZE;
*mmio64 = SPAPR_PCI_BASE + (index + 1) * SPAPR_PCI_MEM64_WIN_SIZE;

*nv2gpa = SPAPR_PCI_NV2RAM64_WIN_BASE + index * SPAPR_PCI_NV2RAM64_WIN_SIZE;
*nv2atsd = SPAPR_PCI_NV2ATSD_WIN_BASE + index * SPAPR_PCI_NV2ATSD_WIN_SIZE;
return true;
}

Expand Down Expand Up @@ -4963,16 +4956,12 @@ DEFINE_SPAPR_MACHINE(4_1, "4.1", false);
static bool phb_placement_4_0(SpaprMachineState *spapr, uint32_t index,
uint64_t *buid, hwaddr *pio,
hwaddr *mmio32, hwaddr *mmio64,
unsigned n_dma, uint32_t *liobns,
hwaddr *nv2gpa, hwaddr *nv2atsd, Error **errp)
unsigned n_dma, uint32_t *liobns, Error **errp)
{
if (!spapr_phb_placement(spapr, index, buid, pio, mmio32, mmio64, n_dma,
liobns, nv2gpa, nv2atsd, errp)) {
liobns, errp)) {
return false;
}

*nv2gpa = 0;
*nv2atsd = 0;
return true;
}
static void spapr_machine_4_0_class_options(MachineClass *mc)
Expand Down Expand Up @@ -5137,8 +5126,7 @@ DEFINE_SPAPR_MACHINE(2_8, "2.8", false);
static bool phb_placement_2_7(SpaprMachineState *spapr, uint32_t index,
uint64_t *buid, hwaddr *pio,
hwaddr *mmio32, hwaddr *mmio64,
unsigned n_dma, uint32_t *liobns,
hwaddr *nv2gpa, hwaddr *nv2atsd, Error **errp)
unsigned n_dma, uint32_t *liobns, Error **errp)
{
/* Legacy PHB placement for pseries-2.7 and earlier machine types */
const uint64_t base_buid = 0x800000020000000ULL;
Expand Down Expand Up @@ -5183,8 +5171,6 @@ static bool phb_placement_2_7(SpaprMachineState *spapr, uint32_t index,
* window into contiguous 32-bit and 64-bit windows
*/

*nv2gpa = 0;
*nv2atsd = 0;
return true;
}

Expand Down
49 changes: 8 additions & 41 deletions hw/ppc/spapr_numa.c
View file
Expand Up @@ -108,20 +108,6 @@ static bool spapr_numa_is_symmetrical(MachineState *ms)
return true;
}

/*
* NVLink2-connected GPU RAM needs to be placed on a separate NUMA node.
* We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is
* called from vPHB reset handler so we initialize the counter here.
* If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM
* must be equally distant from any other node.
* The final value of spapr->gpu_numa_id is going to be written to
* max-associativity-domains in spapr_build_fdt().
*/
unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState *machine)
{
return MAX(1, machine->numa_state->num_nodes);
}

/*
* This function will translate the user distances into
* what the kernel understand as possible values: 10
Expand Down Expand Up @@ -277,7 +263,7 @@ static void spapr_numa_FORM1_affinity_init(SpaprMachineState *spapr,
{
SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
int nb_numa_nodes = machine->numa_state->num_nodes;
int i, j, max_nodes_with_gpus;
int i, j;

/*
* For all associativity arrays: first position is the size,
Expand All @@ -293,17 +279,7 @@ static void spapr_numa_FORM1_affinity_init(SpaprMachineState *spapr,
spapr->FORM1_assoc_array[i][FORM1_DIST_REF_POINTS] = cpu_to_be32(i);
}

/*
* Initialize NVLink GPU associativity arrays. We know that
* the first GPU will take the first available NUMA id, and
* we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine.
* At this point we're not sure if there are GPUs or not, but
* let's initialize the associativity arrays and allow NVLink
* GPUs to be handled like regular NUMA nodes later on.
*/
max_nodes_with_gpus = nb_numa_nodes + NVGPU_MAX_NUM;

for (i = nb_numa_nodes; i < max_nodes_with_gpus; i++) {
for (i = nb_numa_nodes; i < nb_numa_nodes; i++) {
spapr->FORM1_assoc_array[i][0] = cpu_to_be32(FORM1_DIST_REF_POINTS);

for (j = 1; j < FORM1_DIST_REF_POINTS; j++) {
Expand Down Expand Up @@ -345,10 +321,6 @@ static void spapr_numa_FORM2_affinity_init(SpaprMachineState *spapr)
* CPUs will write an additional 'vcpu_id' on top of the arrays
* being initialized here. 'numa_id' is represented by the
* index 'i' of the loop.
*
* Given that this initialization is also valid for GPU associativity
* arrays, handle everything in one single step by populating the
* arrays up to NUMA_NODES_MAX_NUM.
*/
for (i = 0; i < NUMA_NODES_MAX_NUM; i++) {
spapr->FORM2_assoc_array[i][0] = cpu_to_be32(1);
Expand Down Expand Up @@ -461,16 +433,14 @@ static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr,
{
MachineState *ms = MACHINE(spapr);
SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr);
uint32_t number_nvgpus_nodes = spapr->gpu_numa_id -
spapr_numa_initial_nvgpu_numa_id(ms);
uint32_t refpoints[] = {
cpu_to_be32(0x4),
cpu_to_be32(0x3),
cpu_to_be32(0x2),
cpu_to_be32(0x1),
};
uint32_t nr_refpoints = ARRAY_SIZE(refpoints);
uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes;
uint32_t maxdomain = ms->numa_state->num_nodes;
uint32_t maxdomains[] = {
cpu_to_be32(4),
cpu_to_be32(maxdomain),
Expand All @@ -486,13 +456,12 @@ static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr,
cpu_to_be32(0x4),
cpu_to_be32(0x2),
};
uint32_t legacy_maxdomain = spapr->gpu_numa_id > 1 ? 1 : 0;
uint32_t legacy_maxdomains[] = {
cpu_to_be32(4),
cpu_to_be32(legacy_maxdomain),
cpu_to_be32(legacy_maxdomain),
cpu_to_be32(legacy_maxdomain),
cpu_to_be32(spapr->gpu_numa_id),
cpu_to_be32(0),
cpu_to_be32(0),
cpu_to_be32(0),
cpu_to_be32(maxdomain ? maxdomain : 1),
};

G_STATIC_ASSERT(sizeof(legacy_refpoints) <= sizeof(refpoints));
Expand Down Expand Up @@ -581,8 +550,6 @@ static void spapr_numa_FORM2_write_rtas_dt(SpaprMachineState *spapr,
void *fdt, int rtas)
{
MachineState *ms = MACHINE(spapr);
uint32_t number_nvgpus_nodes = spapr->gpu_numa_id -
spapr_numa_initial_nvgpu_numa_id(ms);

/*
* In FORM2, ibm,associativity-reference-points will point to
Expand All @@ -596,7 +563,7 @@ static void spapr_numa_FORM2_write_rtas_dt(SpaprMachineState *spapr,
*/
uint32_t refpoints[] = { cpu_to_be32(1) };

uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes;
uint32_t maxdomain = ms->numa_state->num_nodes;
uint32_t maxdomains[] = { cpu_to_be32(1), cpu_to_be32(maxdomain) };

_FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points",
Expand Down
19 changes: 0 additions & 19 deletions hw/ppc/spapr_pci.c
View file
Expand Up @@ -1443,8 +1443,6 @@ static int spapr_dt_pci_device(SpaprPhbState *sphb, PCIDevice *dev,
_FDT(fdt_setprop_cell(fdt, offset, "ibm,pci-config-space-type", 0x1));
}

spapr_phb_nvgpu_populate_pcidev_dt(dev, fdt, offset, sphb);

if (!IS_PCI_BRIDGE(dev)) {
/* Properties only for non-bridges */
uint32_t min_grant = pci_default_read_config(dev, PCI_MIN_GNT, 1);
Expand Down Expand Up @@ -1757,8 +1755,6 @@ static void spapr_phb_unrealize(DeviceState *dev)
int i;
const unsigned windows_supported = spapr_phb_windows_supported(sphb);

spapr_phb_nvgpu_free(sphb);

if (sphb->msi) {
g_hash_table_unref(sphb->msi);
sphb->msi = NULL;
Expand Down Expand Up @@ -2069,14 +2065,8 @@ void spapr_phb_dma_reset(SpaprPhbState *sphb)
static void spapr_phb_reset(DeviceState *qdev)
{
SpaprPhbState *sphb = SPAPR_PCI_HOST_BRIDGE(qdev);
Error *err = NULL;

spapr_phb_dma_reset(sphb);
spapr_phb_nvgpu_free(sphb);
spapr_phb_nvgpu_setup(sphb, &err);
if (err) {
error_report_err(err);
}

/* Reset the IOMMU state */
object_child_foreach(OBJECT(qdev), spapr_phb_children_reset, NULL);
Expand Down Expand Up @@ -2112,8 +2102,6 @@ static Property spapr_phb_properties[] = {
pre_2_8_migration, false),
DEFINE_PROP_BOOL("pcie-extended-configuration-space", SpaprPhbState,
pcie_ecs, true),
DEFINE_PROP_UINT64("gpa", SpaprPhbState, nv2_gpa_win_addr, 0),
DEFINE_PROP_UINT64("atsd", SpaprPhbState, nv2_atsd_win_addr, 0),
DEFINE_PROP_BOOL("pre-5.1-associativity", SpaprPhbState,
pre_5_1_assoc, false),
DEFINE_PROP_END_OF_LIST(),
Expand Down Expand Up @@ -2362,7 +2350,6 @@ int spapr_dt_phb(SpaprMachineState *spapr, SpaprPhbState *phb,
};
SpaprTceTable *tcet;
SpaprDrc *drc;
Error *err = NULL;

/* Start populating the FDT */
_FDT(bus_off = fdt_add_subnode(fdt, 0, phb->dtbusname));
Expand Down Expand Up @@ -2443,12 +2430,6 @@ int spapr_dt_phb(SpaprMachineState *spapr, SpaprPhbState *phb,
return ret;
}

spapr_phb_nvgpu_populate_dt(phb, fdt, bus_off, &err);
if (err) {
error_report_err(err);
}
spapr_phb_nvgpu_ram_populate_dt(phb, fdt);

return 0;
}

Expand Down

0 comments on commit 44fa20c

Please sign in to comment.