hypervisor: arm64: implement support for PA range of up to 48 bits

We currently support 3 levels of page tables for a 39 bits PA range
on ARM. This patch implements support for 4 level page tables on
AArch64, for PA ranges from 40 to 48 bits. This will allow to use
Jailhouse on more targets.

Signed-off-by: Antonios Motakis <antonios.motakis@huawei.com>

hypervisor/arch/arm/include/asm/paging.h

@@ -31,11 +31,13 @@
  * by IPA[20:12].
  * This would allows to cover a 4GB memory map by using 4 concatenated level-2
  * page tables and thus provide better table walk performances.
- * For the moment, the core doesn't allow to use concatenated pages, so we will
- * use three levels instead, starting at level 1.
+ * For the moment, we will implement the first level for AArch32 using only
+ * one level.
  *
- * TODO: add a "u32 concatenated" field to the paging struct
+ * TODO: implement larger PARange support for AArch32
  */
+#define ARM_CELL_ROOT_PT_SZ	1
+
 #if MAX_PAGE_TABLE_LEVELS < 3
 #define T0SZ			0
 #define SL0			0
@@ -170,6 +172,15 @@
 
 typedef u64 *pt_entry_t;
 
+extern unsigned int cpu_parange;
+
+/* cpu_parange initialized in arch_paging_init */
+static inline unsigned int get_cpu_parange(void)
+{
+	/* TODO: implement proper PARange support on AArch32 */
+	return 39;
+}
+
 /* Only executed on hypervisor paging struct changes */
 static inline void arch_paging_flush_page_tlbs(unsigned long page_addr)
 {

hypervisor/arch/arm/include/asm/paging_modes.h

@@ -16,6 +16,7 @@
 
 /* Long-descriptor paging */
 extern const struct paging arm_paging[];
+extern const struct paging *cell_paging;
 
 #define hv_paging	arm_paging
 

hypervisor/arch/arm/mmu_cell.c

@@ -57,8 +57,13 @@ unsigned long arch_paging_gphys2phys(struct per_cpu *cpu_data,
 
 int arch_mmu_cell_init(struct cell *cell)
 {
-	cell->arch.mm.root_paging = hv_paging;
-	cell->arch.mm.root_table = page_alloc(&mem_pool, 1, 0);
+	if (!get_cpu_parange())
+		return trace_error(-EINVAL);
+
+	cell->arch.mm.root_paging = cell_paging;
+	cell->arch.mm.root_table =
+		page_alloc(&mem_pool, ARM_CELL_ROOT_PT_SZ, 1);
+
 	if (!cell->arch.mm.root_table)
 		return -ENOMEM;
 
@@ -67,7 +72,7 @@ int arch_mmu_cell_init(struct cell *cell)
 
 void arch_mmu_cell_destroy(struct cell *cell)
 {
-	page_free(&mem_pool, cell->arch.mm.root_table, 1);
+	page_free(&mem_pool, cell->arch.mm.root_table, ARM_CELL_ROOT_PT_SZ);
 }
 
 int arch_mmu_cpu_cell_init(struct per_cpu *cpu_data)
@@ -77,6 +82,11 @@ int arch_mmu_cpu_cell_init(struct per_cpu *cpu_data)
 	u64 vttbr = 0;
 	u32 vtcr = VTCR_CELL;
 
+	/* We share page tables between CPUs, so we need to check
+	 * that all CPUs support the same PARange. */
+	if (cpu_parange != get_cpu_parange())
+		return trace_error(-EINVAL);
+
 	if (cell->id > 0xff) {
 		panic_printk("No cell ID available\n");
 		return -E2BIG;

hypervisor/arch/arm/paging.c

@@ -12,6 +12,8 @@
 
 #include <jailhouse/paging.h>
 
+unsigned int cpu_parange = 0;
+
 static bool arm_entry_valid(pt_entry_t entry, unsigned long flags)
 {
 	// FIXME: validate flags!
@@ -40,6 +42,20 @@ static bool arm_page_table_empty(page_table_t page_table)
 	return true;
 }
 
+#if MAX_PAGE_TABLE_LEVELS > 3
+static pt_entry_t arm_get_l0_entry(page_table_t page_table, unsigned long virt)
+{
+	return &page_table[(virt & L0_VADDR_MASK) >> 39];
+}
+
+static unsigned long arm_get_l0_phys(pt_entry_t pte, unsigned long virt)
+{
+	if ((*pte & PTE_TABLE_FLAGS) == PTE_TABLE_FLAGS)
+		return INVALID_PHYS_ADDR;
+	return (*pte & PTE_L0_BLOCK_ADDR_MASK) | (virt & BLOCK_512G_VADDR_MASK);
+}
+#endif
+
 #if MAX_PAGE_TABLE_LEVELS > 2
 static pt_entry_t arm_get_l1_entry(page_table_t page_table, unsigned long virt)
 {
@@ -59,6 +75,18 @@ static unsigned long arm_get_l1_phys(pt_entry_t pte, unsigned long virt)
 }
 #endif
 
+static pt_entry_t arm_get_l1_alt_entry(page_table_t page_table, unsigned long virt)
+{
+	return &page_table[(virt & BIT_MASK(48,30)) >> 30];
+}
+
+static unsigned long arm_get_l1_alt_phys(pt_entry_t pte, unsigned long virt)
+{
+	if ((*pte & PTE_TABLE_FLAGS) == PTE_TABLE_FLAGS)
+		return INVALID_PHYS_ADDR;
+	return (*pte & BIT_MASK(48,30)) | (virt & BIT_MASK(29,0));
+}
+
 static pt_entry_t arm_get_l2_entry(page_table_t page_table, unsigned long virt)
 {
 	return &page_table[(virt & L2_VADDR_MASK) >> 21];
@@ -110,6 +138,18 @@ static unsigned long arm_get_l3_phys(pt_entry_t pte, unsigned long virt)
 		.page_table_empty = arm_page_table_empty,
 
 const struct paging arm_paging[] = {
+#if MAX_PAGE_TABLE_LEVELS > 3
+	{
+		ARM_PAGING_COMMON
+		/* No block entries for level 0! */
+		.page_size = 0,
+		.get_entry = arm_get_l0_entry,
+		.get_phys = arm_get_l0_phys,
+
+		.set_next_pt = arm_set_l12_table,
+		.get_next_pt = arm_get_l12_table,
+	},
+#endif
 #if MAX_PAGE_TABLE_LEVELS > 2
 	{
 		ARM_PAGING_COMMON
@@ -144,6 +184,47 @@ const struct paging arm_paging[] = {
 	}
 };
 
+const struct paging arm_s2_paging_alt[] = {
+	{
+		ARM_PAGING_COMMON
+		.page_size = 0,
+		.get_entry = arm_get_l1_alt_entry,
+		.get_phys = arm_get_l1_alt_phys,
+
+		.set_next_pt = arm_set_l12_table,
+		.get_next_pt = arm_get_l12_table,
+	},
+	{
+		ARM_PAGING_COMMON
+		/* Block entry: 2MB */
+		.page_size = 2 * 1024 * 1024,
+		.get_entry = arm_get_l2_entry,
+		.set_terminal = arm_set_l2_block,
+		.get_phys = arm_get_l2_phys,
+
+		.set_next_pt = arm_set_l12_table,
+		.get_next_pt = arm_get_l12_table,
+	},
+	{
+		ARM_PAGING_COMMON
+		/* Page entry: 4kB */
+		.page_size = 4 * 1024,
+		.get_entry = arm_get_l3_entry,
+		.set_terminal = arm_set_l3_page,
+		.get_phys = arm_get_l3_phys,
+	}
+};
+
+const struct paging *cell_paging;
+
 void arch_paging_init(void)
 {
+	cpu_parange = get_cpu_parange();
+
+	if (cpu_parange < 44)
+		/* 4 level page tables not supported for stage 2.
+		 * We need to use multiple consecutive pages for L1 */
+		cell_paging = arm_s2_paging_alt;
+	else
+		cell_paging = arm_paging;
 }

hypervisor/arch/arm64/entry.S

@@ -66,7 +66,7 @@ el2_entry:
 	msr	vbar_el2, x1
 
 	/* enable temporary mmu mappigns for early initialization */
-	ldr	x0, =bootstrap_pt_l1
+	ldr	x0, =bootstrap_pt_l0
 	bl	enable_mmu_el2
 
 	mov	x0, x17		/* preserved cpuid, will be passed to entry */
@@ -122,11 +122,11 @@ enable_mmu_el2:
 	 * We should think what hw configuration we support by one instance of
 	 * the hypervisor and choose Inner or Outter sharable domain.
 	 */
-	ldr	x1, =(T0SZ | (TCR_RGN_WB_WA << TCR_IRGN0_SHIFT)		\
-			   | (TCR_RGN_WB_WA << TCR_ORGN0_SHIFT)		\
-			   | (TCR_INNER_SHAREABLE << TCR_SH0_SHIFT)	\
-			   | (TCR_PS_40B << TCR_PS_SHIFT)		\
-			   | TCR_EL2_RES1)
+	ldr	x1, =(T0SZ(48) | (TCR_RGN_WB_WA << TCR_IRGN0_SHIFT)	\
+			       | (TCR_RGN_WB_WA << TCR_ORGN0_SHIFT)	\
+			       | (TCR_INNER_SHAREABLE << TCR_SH0_SHIFT)\
+			       | (TCR_PS_48B << TCR_PS_SHIFT)		\
+			       | TCR_EL2_RES1)
 	msr	tcr_el2, x1
 
 	msr	ttbr0_el2, x0
@@ -149,8 +149,32 @@ enable_mmu_el2:
  * we set up the final EL2 page tables.
  */
 .align 12
-bootstrap_pt_l1:
+bootstrap_pt_l0:
 	addr = 0
+	blk_sz = 1 << 39
+        .rept 512
+	.if (addr >> 39) == (UART_BASE >> 39)
+        .quad bootstrap_pt_l1_uart + PTE_TABLE_FLAGS
+	.else
+	.if (addr >> 39) == (JAILHOUSE_BASE >> 39)
+        .quad bootstrap_pt_l1 + PTE_TABLE_FLAGS
+	.else
+        .quad 0
+	.endif
+	.endif
+	addr = addr + blk_sz
+        .endr
+bootstrap_pt_l1:
+#if (JAILHOUSE_BASE >> 39) != (UART_BASE)
+	addr = JAILHOUSE_BASE & ~((1 << 39) - 1)
+	blk_sz = 1 << 30
+        .rept 512
+        .quad addr | PAGE_DEFAULT_FLAGS
+	addr = addr + blk_sz
+        .endr
+#endif
+bootstrap_pt_l1_uart:
+	addr = UART_BASE & ~((1 << 39) - 1)
 	blk_sz = 1 << 30
         .rept 512
 	.if (addr ^ UART_BASE) >> 30

hypervisor/arch/arm64/include/asm/paging.h

@@ -25,24 +25,18 @@
  * native page size. AArch64 also supports 4 levels of page tables, numbered
  * L0-3, while AArch32 supports only 3 levels numbered L1-3.
  *
- * Otherwise, the page table format is identical. By setting the TCR registers
- * appropriately, for 4Kb page tables and starting address translations from
- * level 1, we can use the same page tables and page table generation code that
- * we use on AArch64.
- *
- * This gives us 39 addressable bits for the moment.
- * AARCH64_TODO: implement 4 level page tables, different granule sizes.
+ * We currently only implement 4Kb granule size for the page tables.
+ * We support physical address ranges of up to 48 bits.
  */
 
 #define PAGE_SHIFT		12
 #define PAGE_SIZE		(1 << PAGE_SHIFT)
 #define PAGE_MASK		~(PAGE_SIZE - 1)
 #define PAGE_OFFS_MASK		(PAGE_SIZE - 1)
 
-#define MAX_PAGE_TABLE_LEVELS	3
+#define MAX_PAGE_TABLE_LEVELS	4
 
-#define T0SZ			(64 - 39)
-#define SL0			01
+#define L0_VADDR_MASK		BIT_MASK(47, 39)
 #define L1_VADDR_MASK		BIT_MASK(38, 30)
 #define L2_VADDR_MASK		BIT_MASK(29, 21)
 
@@ -83,11 +77,13 @@
  */
 #define PTE_TABLE_FLAGS		0x3
 
-#define PTE_L1_BLOCK_ADDR_MASK	BIT_MASK(39, 30)
-#define PTE_L2_BLOCK_ADDR_MASK	BIT_MASK(39, 21)
-#define PTE_TABLE_ADDR_MASK	BIT_MASK(39, 12)
-#define PTE_PAGE_ADDR_MASK	BIT_MASK(39, 12)
+#define PTE_L0_BLOCK_ADDR_MASK	BIT_MASK(47, 39)
+#define PTE_L1_BLOCK_ADDR_MASK	BIT_MASK(47, 30)
+#define PTE_L2_BLOCK_ADDR_MASK	BIT_MASK(47, 21)
+#define PTE_TABLE_ADDR_MASK	BIT_MASK(47, 12)
+#define PTE_PAGE_ADDR_MASK	BIT_MASK(47, 12)
 
+#define BLOCK_512G_VADDR_MASK	BIT_MASK(38, 0)
 #define BLOCK_1G_VADDR_MASK	BIT_MASK(29, 0)
 #define BLOCK_2M_VADDR_MASK	BIT_MASK(20, 0)
 
@@ -103,7 +99,16 @@
 
 #define TCR_EL2_RES1		((1 << 31) | (1 << 23))
 #define VTCR_RES1		((1 << 31))
+#define T0SZ(parange)		(64 - parange)
+#define SL0_L0			2
+#define SL0_L1			1
+#define SL0_L2			0
+#define TCR_PS_32B		0x0
+#define TCR_PS_36B		0x1
 #define TCR_PS_40B		0x2
+#define TCR_PS_42B		0x3
+#define TCR_PS_44B		0x4
+#define TCR_PS_48B		0x5
 #define TCR_RGN_NON_CACHEABLE	0x0
 #define TCR_RGN_WB_WA		0x1
 #define TCR_RGN_WT		0x2
@@ -119,15 +124,6 @@
 #define TCR_SL0_SHIFT		6
 #define TCR_S_SHIFT		4
 
-/* AARCH64_TODO: we statically assume a 40 bit address space. Need to fix this,
- * along with the support for the 0th level page table available in AArch64 */
-#define VTCR_CELL		(T0SZ | SL0 << TCR_SL0_SHIFT		\
-				| (TCR_RGN_WB_WA << TCR_IRGN0_SHIFT)	\
-				| (TCR_RGN_WB_WA << TCR_ORGN0_SHIFT)	\
-				| (TCR_INNER_SHAREABLE << TCR_SH0_SHIFT)\
-				| (TCR_PS_40B << TCR_PS_SHIFT)		\
-				| VTCR_RES1)
-
 /*
  * Hypervisor memory attribute indexes:
  *   0: normal WB, RA, WA, non-transient
@@ -173,6 +169,65 @@
 
 typedef u64 *pt_entry_t;
 
+extern unsigned int cpu_parange;
+
+/* cpu_parange initialized in arch_paging_init */
+static inline unsigned int get_cpu_parange(void)
+{
+	unsigned long id_aa64mmfr0;
+
+	arm_read_sysreg(ID_AA64MMFR0_EL1, id_aa64mmfr0);
+
+	switch (id_aa64mmfr0 & 0xf) {
+	case TCR_PS_32B:
+		return 32;
+	case TCR_PS_36B:
+		return 36;
+	case TCR_PS_40B:
+		return 40;
+	case TCR_PS_42B:
+		return 42;
+	case TCR_PS_44B:
+		return 44;
+	case TCR_PS_48B:
+		return 48;
+	default:
+		return 0;
+	}
+}
+
+/* The size of the cpu_parange, determines from which level we can
+ * start from the S2 translations, and the size of the first level
+ * page table */
+#define T0SZ_CELL		T0SZ(cpu_parange)
+#define SL0_CELL		((cpu_parange >= 44) ? SL0_L0 : SL0_L1)
+#define ARM_CELL_ROOT_PT_SZ				\
+	({ unsigned int ret = 1;			\
+	   if (cpu_parange > 39 && cpu_parange < 44)	\
+		ret = 1 << (cpu_parange - 39);		\
+	   ret; })
+
+/* Just match the host's PARange */
+#define TCR_PS_CELL					\
+	({ unsigned int ret = 0;			\
+	   switch (cpu_parange) {			\
+		case 32: ret = TCR_PS_32B; break;	\
+		case 36: ret = TCR_PS_36B; break;	\
+		case 40: ret = TCR_PS_40B; break;	\
+		case 42: ret = TCR_PS_42B; break;	\
+		case 44: ret = TCR_PS_44B; break;	\
+		case 48: ret = TCR_PS_48B; break;	\
+	   }						\
+	   ret; })
+
+#define VTCR_CELL		(T0SZ_CELL | (SL0_CELL << TCR_SL0_SHIFT)\
+				| (TCR_RGN_WB_WA << TCR_IRGN0_SHIFT)	\
+				| (TCR_RGN_WB_WA << TCR_ORGN0_SHIFT)	\
+				| (TCR_INNER_SHAREABLE << TCR_SH0_SHIFT)\
+				| (TCR_PS_CELL << TCR_PS_SHIFT)		\
+				| VTCR_RES1)
+
+
 /* Only executed on hypervisor paging struct changes */
 static inline void arch_paging_flush_page_tlbs(unsigned long page_addr)
 {