amd64: store pcids pmap data in pcpu zone

This change eliminates the struct pmap_pcid array embedded into struct pmap and sized by MAXCPU, which would bloat with MAXCPU increase. Also it removes false sharing of cache lines, since the array elements are mostly locally accessed by corresponding CPUs. Suggested by: mjg Reviewed by: markj Tested by: pho Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D39890
freebsd · May 2, 2023 · 42f722e · 42f722e
1 parent 9c8cbf3
commit 42f722e
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 12 deletions.
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
@@ -384,7 +384,15 @@ pmap_pku_mask_bit(pmap_t pmap)
 #define	VM_PAGE_TO_PV_LIST_LOCK(m)	\
 			PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
 
-struct pmap kernel_pmap_store;
+/*
+ * Statically allocate kernel pmap memory.  However, memory for
+ * pm_pcids is obtained after the dynamic allocator is operational.
+ * Initialize it with a non-canonical pointer to catch early accesses
+ * regardless of the active mapping.
+ */
+struct pmap kernel_pmap_store = {
+	.pm_pcidp = (void *)0xdeadbeefdeadbeef,
+};
 
 vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
 vm_offset_t virtual_end;	/* VA of last avail page (end of kernel AS) */
@@ -2026,9 +2034,14 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
 
 	/* Initialize TLB Context Id. */
 	if (pmap_pcid_enabled) {
+		kernel_pmap->pm_pcidp = (void *)(uintptr_t)
+		    offsetof(struct pcpu, pc_kpmap_store);
 		for (i = 0; i < MAXCPU; i++) {
-			kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN;
-			kernel_pmap->pm_pcids[i].pm_gen = 1;
+			struct pmap_pcid *pcidp;
+
+			pcidp = zpcpu_get_cpu(kernel_pmap->pm_pcidp, i);
+			pcidp->pm_pcid = PMAP_PCID_KERN;
+			pcidp->pm_gen = 1;
 		}
 
 		/*
@@ -3031,6 +3044,7 @@ pmap_invalidate_ept(pmap_t pmap)
 static inline void
 pmap_invalidate_preipi_pcid(pmap_t pmap)
 {
+	struct pmap_pcid *pcidp;
 	u_int cpuid, i;
 
 	sched_pin();
@@ -3040,8 +3054,10 @@ pmap_invalidate_preipi_pcid(pmap_t pmap)
 		cpuid = 0xffffffff;	/* An impossible value */
 
 	CPU_FOREACH(i) {
-		if (cpuid != i)
-			pmap->pm_pcids[i].pm_gen = 0;
+		if (cpuid != i) {
+			pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);
+			pcidp->pm_gen = 0;
+		}
 	}
 
 	/*
@@ -4192,7 +4208,7 @@ pmap_pinit_pcids(pmap_t pmap, uint32_t pcid, int gen)
 	int i;
 
 	CPU_FOREACH(i) {
-		pcidp = &pmap->pm_pcids[i];
+		pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i);
 		pcidp->pm_pcid = pcid;
 		pcidp->pm_gen = gen;
 	}
@@ -4215,6 +4231,7 @@ pmap_pinit0(pmap_t pmap)
 	TAILQ_INIT(&pmap->pm_pvchunk);
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 	pmap->pm_flags = pmap_flags;
+	pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK);
 	pmap_pinit_pcids(pmap, PMAP_PCID_KERN + 1, 1);
 	pmap_activate_boot(pmap);
 	td = curthread;
@@ -4398,6 +4415,9 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)
 	pmap->pm_pmltop = (pml5_entry_t *)PHYS_TO_DMAP(pmltop_phys);
 
 	if (pmap_pcid_enabled) {
+		if (pmap->pm_pcidp == NULL)
+			pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8,
+			    M_WAITOK);
 		pmap_pinit_pcids(pmap, PMAP_PCID_NONE, 0);
 	}
 	pmap->pm_cr3 = PMAP_NO_CR3;	/* initialize to an invalid value */
@@ -9993,11 +10013,11 @@ pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid)
 		PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK);
 		old_pmap = PCPU_GET(curpmap);
 		MPASS(old_pmap->pm_ucr3 != PMAP_NO_CR3);
-		old_pcidp = &old_pmap->pm_pcids[cpuid];
+		old_pcidp = zpcpu_get_cpu(old_pmap->pm_pcidp, cpuid);
 		old_pcidp->pm_gen = 0;
 	}
 
-	pcidp = &pmap->pm_pcids[cpuid];
+	pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);
 	cached = pmap_pcid_alloc_checked(pmap, pcidp);
 	cr3 = rcr3();
 	if ((cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)
@@ -10027,7 +10047,7 @@ pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap,
 	KASSERT((read_rflags() & PSL_I) == 0,
 	    ("PCID needs interrupts disabled in pmap_activate_sw()"));
 
-	pcidp = &pmap->pm_pcids[cpuid];
+	pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid);
 	cached = pmap_pcid_alloc_checked(pmap, pcidp);
 	cr3 = rcr3();
 	if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3)

diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
@@ -35,6 +35,7 @@
 #ifndef _MACHINE_PCPU_H_
 #define	_MACHINE_PCPU_H_
 
+#include <machine/_pmap.h>
 #include <machine/segments.h>
 #include <machine/tss.h>
 
@@ -101,7 +102,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line");
 	uint64_t pc_ucr3_load_mask;					\
 	u_int	pc_small_core;						\
 	u_int	pc_pcid_invlpg_workaround;				\
-	char	__pad[2908]		/* pad to UMA_PCPU_ALLOC_SIZE */
+	struct pmap_pcid pc_kpmap_store;				\
+	char	__pad[2900]		/* pad to UMA_PCPU_ALLOC_SIZE */
 
 #define	PC_DBREG_CMD_NONE	0
 #define	PC_DBREG_CMD_LOAD	1

diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
@@ -286,6 +286,7 @@
 
 #ifndef LOCORE
 
+#include <sys/kassert.h>
 #include <sys/queue.h>
 #include <sys/_cpuset.h>
 #include <sys/_lock.h>
@@ -390,7 +391,7 @@ struct pmap {
 	long			pm_eptgen;	/* EPT pmap generation id */
 	smr_t			pm_eptsmr;
 	int			pm_flags;
-	struct pmap_pcid	pm_pcids[MAXCPU];
+	struct pmap_pcid	*pm_pcidp;
 	struct rangeset		pm_pkru;
 };
 
@@ -537,8 +538,11 @@ pmap_invlpg(pmap_t pmap, vm_offset_t va)
 static __inline uint32_t
 pmap_get_pcid(pmap_t pmap)
 {
+	struct pmap_pcid *pcidp;
+
 	MPASS(pmap_pcid_enabled);
-	return (pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid);
+	pcidp = zpcpu_get(pmap->pm_pcidp);
+	return (pcidp->pm_pcid);
 }
 
 #endif /* _KERNEL */