Skip to content

Commit

Permalink
HV: correct ept page array usage
Browse files Browse the repository at this point in the history 
Currently ept_pages_info[] is initialized with first element only that force
VM of id 0 using SOS EPT pages. This is incorrect for logical partition and
hybrid scenario. Considering SOS_RAM_SIZE and UOS_RAM_SIZE are configured
separately, we should use different ept pages accordingly.

So, the PRE_VM_NUM/SOS_VM_NUM and MAX_POST_VM_NUM macros are introduced to
resolve this issue. The macros would be generated by acrn-config tool when
user configure ACRN for their specific scenario.

One more thing, that when UOS_RAM_SIZE is less then 2GB, the EPT address
range should be (4G + PLATFORM_HI_MMIO_SIZE).

Tracked-On: #4458

Signed-off-by: Victor Sun <victor.sun@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>
  • Loading branch information
@jsun26intel @wenlingz
jsun26intel authored and wenlingz committed Mar 12, 2020
1 parent e9a9984 commit 4c0965d
Show file tree
Hide file tree
Showing 12 changed files with 88 additions and 43 deletions.
6 changes: 3 additions & 3 deletions hypervisor/arch/x86/mmu.c
View file
Expand Up @@ -277,10 +277,10 @@ void init_paging(void)
mmu_modify_or_del((uint64_t *)ppt_mmu_pml4_addr, round_pde_down(hv_hva),
round_pde_up((uint64_t)&ld_text_end) - round_pde_down(hv_hva), 0UL,
PAGE_NX, &ppt_mem_ops, MR_MODIFY);

#if (SOS_VM_NUM == 1)
mmu_modify_or_del((uint64_t *)ppt_mmu_pml4_addr, (uint64_t)get_reserve_sworld_memory_base(),
TRUSTY_RAM_SIZE * (CONFIG_MAX_VM_NUM - 1U), PAGE_USER, 0UL, &ppt_mem_ops, MR_MODIFY);

TRUSTY_RAM_SIZE * MAX_POST_VM_NUM, PAGE_USER, 0UL, &ppt_mem_ops, MR_MODIFY);
#endif
/* Enable paging */
enable_paging();

Expand Down
75 changes: 43 additions & 32 deletions hypervisor/arch/x86/page.c
View file
Expand Up @@ -10,7 +10,6 @@
#include <mmu.h>
#include <trusty.h>
#include <vtd.h>
#include <vm_configurations.h>
#include <security.h>
#include <vm.h>

Expand Down Expand Up @@ -78,37 +77,33 @@ const struct memory_ops ppt_mem_ops = {
.recover_exe_right = nop_recover_exe_right,
};

static struct page sos_vm_pml4_pages[PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pdpt_pages[PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pd_pages[PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pt_pages[PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pml4_pages[SOS_VM_NUM][PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pdpt_pages[SOS_VM_NUM][PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pd_pages[SOS_VM_NUM][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];
static struct page sos_vm_pt_pages[SOS_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE))];

/* uos_nworld_pml4_pages[i] is ...... of UOS i (whose vm_id = i +1) */
static struct page uos_nworld_pml4_pages[CONFIG_MAX_VM_NUM - 1U][PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page uos_nworld_pdpt_pages[CONFIG_MAX_VM_NUM - 1U][PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page uos_nworld_pd_pages[CONFIG_MAX_VM_NUM - 1U][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page uos_nworld_pt_pages[CONFIG_MAX_VM_NUM - 1U][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
/* pre_uos_nworld_pml4_pages */
static struct page pre_uos_nworld_pml4_pages[PRE_VM_NUM][PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page pre_uos_nworld_pdpt_pages[PRE_VM_NUM][PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page pre_uos_nworld_pd_pages[PRE_VM_NUM][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page pre_uos_nworld_pt_pages[PRE_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];

static struct page uos_sworld_pgtable_pages[CONFIG_MAX_VM_NUM - 1U][TRUSTY_PGTABLE_PAGE_NUM(TRUSTY_RAM_SIZE)];
/* post_uos_nworld_pml4_pages */
static struct page post_uos_nworld_pml4_pages[MAX_POST_VM_NUM][PML4_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page post_uos_nworld_pdpt_pages[MAX_POST_VM_NUM][PDPT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page post_uos_nworld_pd_pages[MAX_POST_VM_NUM][PD_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];
static struct page post_uos_nworld_pt_pages[MAX_POST_VM_NUM][PT_PAGE_NUM(EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE))];

static struct page post_uos_sworld_pgtable_pages[MAX_POST_VM_NUM][TRUSTY_PGTABLE_PAGE_NUM(TRUSTY_RAM_SIZE)];
/* pre-assumption: TRUSTY_RAM_SIZE is 2M aligned */
static struct page uos_sworld_memory[CONFIG_MAX_VM_NUM - 1U][TRUSTY_RAM_SIZE >> PAGE_SHIFT] __aligned(MEM_2M);
static struct page post_uos_sworld_memory[MAX_POST_VM_NUM][TRUSTY_RAM_SIZE >> PAGE_SHIFT] __aligned(MEM_2M);

/* ept: extended page table*/
static union pgtable_pages_info ept_pages_info[CONFIG_MAX_VM_NUM] = {
{
.ept = {
.top_address_space = EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE),
.nworld_pml4_base = sos_vm_pml4_pages,
.nworld_pdpt_base = sos_vm_pdpt_pages,
.nworld_pd_base = sos_vm_pd_pages,
.nworld_pt_base = sos_vm_pt_pages,
},
},
};
static union pgtable_pages_info ept_pages_info[CONFIG_MAX_VM_NUM];

void *get_reserve_sworld_memory_base(void)
{
return uos_sworld_memory;
return post_uos_sworld_memory;
}

static inline uint64_t ept_get_default_access_right(void)
Expand Down Expand Up @@ -188,15 +183,31 @@ static inline void ept_recover_exe_right(uint64_t *entry)

void init_ept_mem_ops(struct memory_ops *mem_ops, uint16_t vm_id)
{
if (vm_id != 0U) {
struct acrn_vm *vm = get_vm_from_vmid(vm_id);

if (is_sos_vm(vm)) {
ept_pages_info[vm_id].ept.top_address_space = EPT_ADDRESS_SPACE(CONFIG_SOS_RAM_SIZE);
ept_pages_info[vm_id].ept.nworld_pml4_base = sos_vm_pml4_pages[0U];
ept_pages_info[vm_id].ept.nworld_pdpt_base = sos_vm_pdpt_pages[0U];
ept_pages_info[vm_id].ept.nworld_pd_base = sos_vm_pd_pages[0U];
ept_pages_info[vm_id].ept.nworld_pt_base = sos_vm_pt_pages[0U];
} else if (is_prelaunched_vm(vm)) {
ept_pages_info[vm_id].ept.top_address_space = EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE);
ept_pages_info[vm_id].ept.nworld_pml4_base = uos_nworld_pml4_pages[vm_id - 1U];
ept_pages_info[vm_id].ept.nworld_pdpt_base = uos_nworld_pdpt_pages[vm_id - 1U];
ept_pages_info[vm_id].ept.nworld_pd_base = uos_nworld_pd_pages[vm_id - 1U];
ept_pages_info[vm_id].ept.nworld_pt_base = uos_nworld_pt_pages[vm_id - 1U];
ept_pages_info[vm_id].ept.sworld_pgtable_base = uos_sworld_pgtable_pages[vm_id - 1U];
ept_pages_info[vm_id].ept.sworld_memory_base = uos_sworld_memory[vm_id - 1U];
ept_pages_info[vm_id].ept.nworld_pml4_base = pre_uos_nworld_pml4_pages[vm_id];
ept_pages_info[vm_id].ept.nworld_pdpt_base = pre_uos_nworld_pdpt_pages[vm_id];
ept_pages_info[vm_id].ept.nworld_pd_base = pre_uos_nworld_pd_pages[vm_id];
ept_pages_info[vm_id].ept.nworld_pt_base = pre_uos_nworld_pt_pages[vm_id];
} else {
uint16_t sos_vm_id = (get_sos_vm())->vm_id;
uint16_t page_idx = vmid_2_rel_vmid(sos_vm_id, vm_id) - 1U;

ept_pages_info[vm_id].ept.top_address_space = EPT_ADDRESS_SPACE(CONFIG_UOS_RAM_SIZE);
ept_pages_info[vm_id].ept.nworld_pml4_base = post_uos_nworld_pml4_pages[page_idx];
ept_pages_info[vm_id].ept.nworld_pdpt_base = post_uos_nworld_pdpt_pages[page_idx];
ept_pages_info[vm_id].ept.nworld_pd_base = post_uos_nworld_pd_pages[page_idx];
ept_pages_info[vm_id].ept.nworld_pt_base = post_uos_nworld_pt_pages[page_idx];
ept_pages_info[vm_id].ept.sworld_pgtable_base = post_uos_sworld_pgtable_pages[page_idx];
ept_pages_info[vm_id].ept.sworld_memory_base = post_uos_sworld_memory[page_idx];
mem_ops->get_sworld_memory_base = ept_get_sworld_memory_base;
}
mem_ops->info = &ept_pages_info[vm_id];
Expand All @@ -214,7 +225,7 @@ void init_ept_mem_ops(struct memory_ops *mem_ops, uint16_t vm_id)
mem_ops->tweak_exe_right = ept_tweak_exe_right;
mem_ops->recover_exe_right = ept_recover_exe_right;
/* For RTVM, build 4KB page mapping in EPT */
if (is_rt_vm(get_vm_from_vmid(vm_id))) {
if (is_rt_vm(vm)) {
mem_ops->large_page_enabled = false;
}
} else {
Expand Down
2 changes: 1 addition & 1 deletion hypervisor/arch/x86/vtd.c
View file
Expand Up @@ -20,7 +20,7 @@
#include <timer.h>
#include <logmsg.h>
#include <board.h>
#include <vm_configurations.h>
#include <vm_config.h>
#include <pci.h>

#define DBG_IOMMU 0
Expand Down
1 change: 1 addition & 0 deletions hypervisor/include/arch/x86/mmu.h
View file
Expand Up @@ -40,6 +40,7 @@
#define MEM_1M (MEM_1K * 1024U)
#define MEM_2M (MEM_1M * 2U)
#define MEM_1G (MEM_1M * 1024U)
#define MEM_2G (1024UL * 1024UL * 1024UL * 2UL)

#ifndef ASSEMBLER

Expand Down
4 changes: 3 additions & 1 deletion hypervisor/include/arch/x86/page.h
View file
Expand Up @@ -29,7 +29,9 @@
* - Guest OS won't re-program device MMIO bars to the address not covered by
* this EPT_ADDRESS_SPACE.
*/
#define EPT_ADDRESS_SPACE(size) (((size) != 0UL) ? ((size) + PLATFORM_LO_MMIO_SIZE + PLATFORM_HI_MMIO_SIZE) : 0UL)
#define EPT_ADDRESS_SPACE(size) ((size > MEM_2G) ? \
((size) + PLATFORM_LO_MMIO_SIZE + PLATFORM_HI_MMIO_SIZE) \
: (MEM_2G + PLATFORM_LO_MMIO_SIZE + PLATFORM_HI_MMIO_SIZE))

#define TRUSTY_PML4_PAGE_NUM(size) (1UL)
#define TRUSTY_PDPT_PAGE_NUM(size) (1UL)
Expand Down
2 changes: 2 additions & 0 deletions hypervisor/include/arch/x86/vm_config.h
View file
Expand Up @@ -14,6 +14,8 @@
#include <vm_configurations.h>
#include <sgx.h>

#define CONFIG_MAX_VM_NUM (PRE_VM_NUM + SOS_VM_NUM + MAX_POST_VM_NUM)

#define AFFINITY_CPU(n) (1U << (n))
#define MAX_VCPUS_PER_VM MAX_PCPU_NUM
#define MAX_VUART_NUM_PER_VM 2U
Expand Down
1 change: 0 additions & 1 deletion hypervisor/include/debug/profiling_internal.h
View file
Expand Up @@ -11,7 +11,6 @@

#include <vcpu.h>
#include <vm_config.h>
#include <vm_configurations.h>

#define MAX_MSR_LIST_NUM 15U
#define MAX_PROFILING_MSR_STORE_NUM 1
Expand Down
8 changes: 7 additions & 1 deletion hypervisor/scenarios/hybrid/vm_configurations.h
View file
Expand Up @@ -13,7 +13,13 @@
#define DM_OWNED_GUEST_FLAG_MASK (GUEST_FLAG_SECURE_WORLD_ENABLED | GUEST_FLAG_LAPIC_PASSTHROUGH | \
GUEST_FLAG_RT | GUEST_FLAG_IO_COMPLETION_POLLING)

#define CONFIG_MAX_VM_NUM (3U + CONFIG_MAX_KATA_VM_NUM)
/* SOS_VM_NUM can only be 0U or 1U;
* When SOS_VM_NUM is 0U, MAX_POST_VM_NUM must be 0U too;
* MAX_POST_VM_NUM must be bigger than CONFIG_MAX_KATA_VM_NUM;
*/
#define PRE_VM_NUM 1U
#define SOS_VM_NUM 1U
#define MAX_POST_VM_NUM 1U

#define VM0_CONFIG_VCPU_AFFINITY {AFFINITY_CPU(3U)}
#define VM0_CONFIG_MEM_START_HPA 0x100000000UL
Expand Down
8 changes: 7 additions & 1 deletion hypervisor/scenarios/industry/vm_configurations.h
View file
Expand Up @@ -9,7 +9,13 @@

#include <misc_cfg.h>

#define CONFIG_MAX_VM_NUM (3U + CONFIG_MAX_KATA_VM_NUM)
/* SOS_VM_NUM can only be 0U or 1U;
* When SOS_VM_NUM is 0U, MAX_POST_VM_NUM must be 0U too;
* MAX_POST_VM_NUM must be bigger than CONFIG_MAX_KATA_VM_NUM;
*/
#define PRE_VM_NUM 0U
#define SOS_VM_NUM 1U
#define MAX_POST_VM_NUM 2U

/* Bits mask of guest flags that can be programmed by device model. Other bits are set by hypervisor only */
#define DM_OWNED_GUEST_FLAG_MASK (GUEST_FLAG_SECURE_WORLD_ENABLED | GUEST_FLAG_LAPIC_PASSTHROUGH | \
Expand Down
8 changes: 7 additions & 1 deletion hypervisor/scenarios/logical_partition/vm_configurations.h
View file
Expand Up @@ -13,7 +13,13 @@
/* Bits mask of guest flags that can be programmed by device model. Other bits are set by hypervisor only */
#define DM_OWNED_GUEST_FLAG_MASK 0UL

#define CONFIG_MAX_VM_NUM 2U
/* SOS_VM_NUM can only be 0U or 1U;
* When SOS_VM_NUM is 0U, MAX_POST_VM_NUM must be 0U too;
* MAX_POST_VM_NUM must be bigger than CONFIG_MAX_KATA_VM_NUM;
*/
#define PRE_VM_NUM 2U
#define SOS_VM_NUM 0U
#define MAX_POST_VM_NUM 0U

/* The VM CONFIGs like:
* VMX_CONFIG_VCPU_AFFINITY
Expand Down
8 changes: 7 additions & 1 deletion hypervisor/scenarios/sdc/vm_configurations.h
View file
Expand Up @@ -9,7 +9,13 @@

#include <misc_cfg.h>

#define CONFIG_MAX_VM_NUM (2U + CONFIG_MAX_KATA_VM_NUM)
/* SOS_VM_NUM can only be 0U or 1U;
* When SOS_VM_NUM is 0U, MAX_POST_VM_NUM must be 0U too;
* MAX_POST_VM_NUM must be bigger than CONFIG_MAX_KATA_VM_NUM;
*/
#define PRE_VM_NUM 0U
#define SOS_VM_NUM 1U
#define MAX_POST_VM_NUM 2U /* including 1 KATA VM */

/* Bits mask of guest flags that can be programmed by device model. Other bits are set by hypervisor only */
#define DM_OWNED_GUEST_FLAG_MASK (GUEST_FLAG_SECURE_WORLD_ENABLED | GUEST_FLAG_LAPIC_PASSTHROUGH | \
Expand Down
8 changes: 7 additions & 1 deletion hypervisor/scenarios/sdc2/vm_configurations.h
View file
Expand Up @@ -9,7 +9,13 @@

#include <misc_cfg.h>

#define CONFIG_MAX_VM_NUM (4U + CONFIG_MAX_KATA_VM_NUM)
/* SOS_VM_NUM can only be 0U or 1U;
* When SOS_VM_NUM is 0U, MAX_POST_VM_NUM must be 0U too;
* MAX_POST_VM_NUM must be bigger than CONFIG_MAX_KATA_VM_NUM;
*/
#define PRE_VM_NUM 0U
#define SOS_VM_NUM 1U
#define MAX_POST_VM_NUM 3U

/* Bits mask of guest flags that can be programmed by device model. Other bits are set by hypervisor only */
#define DM_OWNED_GUEST_FLAG_MASK (GUEST_FLAG_SECURE_WORLD_ENABLED | GUEST_FLAG_LAPIC_PASSTHROUGH | \
Expand Down

0 comments on commit 4c0965d

Please sign in to comment.