Skip to content

Commit

Permalink
add P2P support
Browse files Browse the repository at this point in the history
  • Loading branch information
@geohot
geohot committed Apr 12, 2024
1 parent 12933b2 commit 1f4613d
Show file tree
Hide file tree
Showing 10 changed files with 177 additions and 943 deletions.
1,038 changes: 123 additions & 915 deletions README.md
View file

Large diffs are not rendered by default.

10 changes: 10 additions & 0 deletions install.sh
View file
@@ -0,0 +1,10 @@
#!/bin/bash
sudo rmmod nvidia nvidia_uvm nvidia_modeset nvidia_drm
sudo rmmod nvidia nvidia_uvm nvidia_modeset nvidia_drm
sudo rmmod nvidia nvidia_uvm nvidia_modeset nvidia_drm
sudo rmmod nvidia nvidia_uvm nvidia_modeset nvidia_drm
set -e
make modules -j$(nproc)
sudo make modules_install -j$(nproc)
sudo depmod
nvidia-smi
4 changes: 2 additions & 2 deletions kernel-open/nvidia/nv-pci.c
View file
Expand Up @@ -177,11 +177,11 @@ static int nv_resize_pcie_bars(struct pci_dev *pci_dev) {
struct pci_host_bridge *host;
#endif

if (NVreg_EnableResizableBar == 0)
/*if (NVreg_EnableResizableBar == 0)
{
nv_printf(NV_DBG_INFO, "NVRM: resizable BAR disabled by regkey, skipping\n");
return 0;
}
}*/

// Check if BAR1 has PCIe rebar capabilities
sizes = pci_rebar_get_possible_sizes(pci_dev, NV_GPU_BAR1);
Expand Down
4 changes: 2 additions & 2 deletions src/nvidia/src/kernel/gpu/bif/kernel_bif.c
View file
Expand Up @@ -709,7 +709,7 @@ _kbifInitRegistryOverrides
NvU32 data32;

// P2P Override
pKernelBif->p2pOverride = BIF_P2P_NOT_OVERRIDEN;
pKernelBif->p2pOverride = 0x11;
if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK)
{
pKernelBif->p2pOverride = data32;
Expand All @@ -718,7 +718,7 @@ _kbifInitRegistryOverrides
}

// P2P force type override
pKernelBif->forceP2PType = NV_REG_STR_RM_FORCE_P2P_TYPE_DEFAULT;
pKernelBif->forceP2PType = NV_REG_STR_RM_FORCE_P2P_TYPE_BAR1P2P;
if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FORCE_P2P_TYPE, &data32) == NV_OK &&
(data32 <= NV_REG_STR_RM_FORCE_P2P_TYPE_MAX))
{
Expand Down
10 changes: 5 additions & 5 deletions src/nvidia/src/kernel/gpu/bus/arch/hopper/kern_bus_gh100.c
View file
Expand Up @@ -765,7 +765,7 @@ kbusCreateP2PMapping_GH100

if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE_BAR1, attributes))
{
return kbusCreateP2PMappingForBar1P2P_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes);
return kbusCreateP2PMappingForBar1P2P_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes);
}

if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes))
Expand Down Expand Up @@ -816,7 +816,7 @@ kbusRemoveP2PMapping_GH100

if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE_BAR1, attributes))
{
return kbusRemoveP2PMappingForBar1P2P_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes);
return kbusRemoveP2PMappingForBar1P2P_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes);
}

if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE, attributes))
Expand Down Expand Up @@ -1448,7 +1448,7 @@ kbusCreateP2PMappingForBar1P2P_GH100
return NV_ERR_NOT_SUPPORTED;
}

if (!kbusIsPcieBar1P2PMappingSupported_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1))
if (!kbusIsPcieBar1P2PMappingSupported_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1))
{
return NV_ERR_NOT_SUPPORTED;
}
Expand Down Expand Up @@ -1871,8 +1871,8 @@ kbusEnableStaticBar1Mapping_GH100
pKernelBus->bar1[gfid].staticBar1.pDmaMemDesc = pDmaMemDesc;
pKernelBus->bar1[gfid].staticBar1.size = bar1MapSize;

NV_PRINTF(LEVEL_INFO, "Static bar1 mapped offset 0x%llx size 0x%llx\n",
bar1Offset, bar1MapSize);
NV_PRINTF(LEVEL_ERROR, "Static bar1 mapped offset 0x%llx size 0x%llx bar1BusAddr 0x%llx\n",
bar1Offset, bar1MapSize, bar1BusAddr);

return NV_OK;

Expand Down
12 changes: 6 additions & 6 deletions src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm107.c
View file
Expand Up @@ -1008,7 +1008,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid)
//
NV_ASSERT(pKernelBus->bar1[gfid].apertureLength <= kbusGetPciBarSize(pKernelBus, 1));

bBar1P2PCapable = kbusIsBar1P2PCapable(pGpu, pKernelBus, gfid);
bBar1P2PCapable = kbusIsBar1P2PCapable_GH100(pGpu, pKernelBus, gfid);

//
// If we need to preserve a console mapping at the start of BAR1, we
Expand Down Expand Up @@ -1094,7 +1094,7 @@ kbusInitBar1_GM107(OBJGPU *pGpu, KernelBus *pKernelBus, NvU32 gfid)
{
// Enable the static BAR1 mapping for the BAR1 P2P
NV_ASSERT_OK_OR_GOTO(rmStatus,
kbusEnableStaticBar1Mapping_HAL(pGpu, pKernelBus, gfid),
kbusEnableStaticBar1Mapping_GH100(pGpu, pKernelBus, gfid),
kbusInitBar1_failed);
}
else
Expand Down Expand Up @@ -3013,15 +3013,15 @@ _kbusMapAperture_GM107
NvU32 swizzId = KMIGMGR_SWIZZID_INVALID;
NvU32 gfid;

if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) &&
/*if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) &&
(memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM))
{
NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
return kbusGetStaticFbAperture_HAL(pGpu, pKernelBus, pMemDesc,
offset, pAperOffset,
pLength, gfid);
}
}*/

// Ensure that the BAR1 VA space is the same across all subdevices
if (IsSLIEnabled(pGpu) && ((mapFlags & BUS_MAP_FB_FLAGS_MAP_UNICAST) == 0))
Expand Down Expand Up @@ -3136,12 +3136,12 @@ _kbusUnmapAperture_GM107
NV_STATUS rmStatus = NV_OK;
VirtMemAllocator *pDma = GPU_GET_DMA(pGpu);

if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) &&
/*if (kbusIsStaticBar1Enabled(pGpu, pKernelBus) &&
(memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM))
{
// No op for the static bar1 mode
return NV_OK;
}
}*/

rmStatus = dmaFreeMapping_HAL(pGpu, pDma, pVAS, aperOffset, pMemDesc, 0, NULL);

Expand Down
5 changes: 5 additions & 0 deletions src/nvidia/src/kernel/gpu/bus/arch/pascal/kern_bus_gp100.c
View file
Expand Up @@ -70,6 +70,11 @@ kbusCreateP2PMapping_GP100
return kbusCreateP2PMappingForMailbox_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, peer0, peer1, attributes);
}

if (FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _PCIE_BAR1, attributes))
{
return kbusCreateP2PMappingForBar1P2P_GH100(pGpu0, pKernelBus0, pGpu1, pKernelBus1, attributes);
}

NV_PRINTF(LEVEL_ERROR, "P2P type %d is not supported\n", DRF_VAL(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, attributes));

return NV_ERR_NOT_SUPPORTED;
Expand Down
12 changes: 6 additions & 6 deletions src/nvidia/src/kernel/gpu/bus/p2p_api.c
View file
Expand Up @@ -574,13 +574,13 @@ p2papiConstruct_IMPL

// setup the p2p resources
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
kbusCreateP2PMapping_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu,
kbusCreateP2PMapping_GH100(pLocalGpu, pLocalKernelBus, pRemoteGpu,
pRemoteKernelBus, &peer1, &peer2,
pP2PApi->attributes));
if (bEgmPeer)
{
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
kbusCreateP2PMapping_HAL(pLocalGpu, pLocalKernelBus, pRemoteGpu,
kbusCreateP2PMapping_GH100(pLocalGpu, pLocalKernelBus, pRemoteGpu,
pRemoteKernelBus, &egmPeer1, &egmPeer2,
pP2PApi->attributes |
DRF_DEF(_P2PAPI, _ATTRIBUTES, _REMOTE_EGM, _YES)));
Expand All @@ -590,13 +590,13 @@ p2papiConstruct_IMPL
(pCallContext->secInfo.privLevel >= RS_PRIV_LEVEL_KERNEL))
{
NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
kbusGetBar1P2PDmaInfo_HAL(pLocalGpu, pRemoteGpu,
kbusGetBar1P2PDmaInfo_GH100(pLocalGpu, pRemoteGpu,
pRemoteKernelBus,
&pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_address,
&pNv503bAllocParams->l2pBar1P2PDmaInfo.dma_size));

NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
kbusGetBar1P2PDmaInfo_HAL(pRemoteGpu, pLocalGpu,
kbusGetBar1P2PDmaInfo_GH100(pRemoteGpu, pLocalGpu,
pLocalKernelBus,
&pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_address,
&pNv503bAllocParams->p2lBar1P2PDmaInfo.dma_size));
Expand Down Expand Up @@ -732,15 +732,15 @@ p2papiDestruct_IMPL
{
// remove any resources associated with this mapping
NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
kbusRemoveP2PMapping_HAL(pLocalGpu, pLocalKernelBus,
kbusRemoveP2PMapping_GH100(pLocalGpu, pLocalKernelBus,
pRemoteGpu, pRemoteKernelBus,
pP2PApi->peerId1, pP2PApi->peerId2,
pP2PApi->attributes), end);
if (!FLD_TEST_DRF(_P2PAPI, _ATTRIBUTES, _LINK_TYPE, _SPA, pP2PApi->attributes) &&
memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pLocalGpu)) &&
memmgrIsLocalEgmEnabled(GPU_GET_MEMORY_MANAGER(pRemoteGpu)))
{
status = kbusRemoveP2PMapping_HAL(pLocalGpu, pLocalKernelBus,
status = kbusRemoveP2PMapping_GH100(pLocalGpu, pLocalKernelBus,
pRemoteGpu, pRemoteKernelBus,
pP2PApi->egmPeerId1, pP2PApi->egmPeerId2,
pP2PApi->attributes |
Expand Down
22 changes: 17 additions & 5 deletions src/nvidia/src/kernel/rmapi/nv_gpu_ops.c
View file
Expand Up @@ -3058,7 +3058,8 @@ nvGpuOpsBuildExternalAllocPtes
NvBool isIndirectPeerSupported,
NvBool isPeerSupported,
NvU32 peerId,
gpuExternalMappingInfo *pGpuExternalMappingInfo
gpuExternalMappingInfo *pGpuExternalMappingInfo,
RmPhysAddr bar1BusAddr
)
{
NV_STATUS status = NV_OK;
Expand Down Expand Up @@ -3253,7 +3254,11 @@ nvGpuOpsBuildExternalAllocPtes
if (nvFieldIsValid32(&pPteFmt->fldAtomicDisable.desc))
nvFieldSetBool(&pPteFmt->fldAtomicDisable, !atomic, pte.v8);

if (aperture == GMMU_APERTURE_PEER) {
gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_NONCOH, pte.v8);
} else {
gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8);
}

if (!isCompressedKind)
{
Expand All @@ -3264,7 +3269,11 @@ nvGpuOpsBuildExternalAllocPtes
}
}

if (aperture == GMMU_APERTURE_PEER)
if (aperture == GMMU_APERTURE_PEER) {
fabricBaseAddress = bar1BusAddr;
}

/*if (aperture == GMMU_APERTURE_PEER)
{
FlaMemory* pFlaMemory = dynamicCast(pMemory, FlaMemory);
nvFieldSet32(&pPteFmt->fldPeerIndex, peerId, pte.v8);
Expand Down Expand Up @@ -3296,7 +3305,7 @@ nvGpuOpsBuildExternalAllocPtes
fabricBaseAddress = knvlinkGetUniqueFabricBaseAddress(pMemDesc->pGpu, pKernelNvlink);
}
}
}
}*/

//
// Both memdescGetPhysAddr() and kgmmuEncodePhysAddr() have pretty high overhead.
Expand Down Expand Up @@ -3451,6 +3460,7 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace,
Memory *pMemory = NULL;
PMEMORY_DESCRIPTOR pMemDesc = NULL;
OBJGPU *pMappingGpu = NULL;
RmPhysAddr bar1BusAddr = 0;
NvU32 peerId = 0;
NvBool isSliSupported = NV_FALSE;
NvBool isPeerSupported = NV_FALSE;
Expand Down Expand Up @@ -3596,6 +3606,8 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace,
&peerId);
if (status != NV_OK)
goto freeGpaMemdesc;

bar1BusAddr = gpumgrGetGpuPhysFbAddr(pAdjustedMemDesc->pGpu);
}

//
Expand Down Expand Up @@ -3673,7 +3685,7 @@ NV_STATUS nvGpuOpsGetExternalAllocPtes(struct gpuAddressSpace *vaSpace,

status = nvGpuOpsBuildExternalAllocPtes(pVAS, pMappingGpu, pAdjustedMemDesc, pMemory, offset, size,
isIndirectPeerSupported, isPeerSupported, peerId,
pGpuExternalMappingInfo);
pGpuExternalMappingInfo, bar1BusAddr);

freeGpaMemdesc:
if (pAdjustedMemDesc != pMemDesc)
Expand Down Expand Up @@ -9322,7 +9334,7 @@ NV_STATUS nvGpuOpsGetChannelResourcePtes(struct gpuAddressSpace *vaSpace,

status = nvGpuOpsBuildExternalAllocPtes(pVAS, pMappingGpu, pMemDesc, NULL,
offset, size, NV_FALSE, NV_FALSE,
0, pGpuExternalMappingInfo);
0, pGpuExternalMappingInfo, 0);

_nvGpuOpsLocksRelease(&acquiredLocks);
threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);
Expand Down
3 changes: 1 addition & 2 deletions src/nvidia/src/libraries/mmu/gmmu_fmt.c
View file
Expand Up @@ -124,9 +124,8 @@ gmmuFmtPtePhysAddrFld
{
case GMMU_APERTURE_SYS_COH:
case GMMU_APERTURE_SYS_NONCOH:
return &pPte->fldAddrSysmem;
case GMMU_APERTURE_PEER:
return &pPte->fldAddrPeer;
return &pPte->fldAddrSysmem;
case GMMU_APERTURE_VIDEO:
return &pPte->fldAddrVidmem;
default:
Expand Down

0 comments on commit 1f4613d

Please sign in to comment.