hv: Add update_vm_vlapic_state API to sync the VM vLAPIC state

This patch introduces vLAPIC state for a VM. The VM vLAPIC state can
be one of the following
 * VM_VLAPIC_X2APIC - All the vCPUs/vLAPICs (Except for those in Disabled mode) of this
	VM use x2APIC mode
 * VM_VLAPIC_XAPIC - All the vCPUs/vLAPICs (Except for those in Disabled mode) of this
	VM use xAPIC mode
 * VM_VLAPIC_DISABLED - All the vCPUs/vLAPICs of this VM are in Disabled mode
 * VM_VLAPIC_TRANSITION - Some of the vCPUs/vLAPICs of this VM (Except for those in Disabled mode)
	are in xAPIC and the others in x2APIC

Upon a vCPU updating the IA32_APIC_BASE MSR to switch LAPIC mode, this
API is called to sync the vLAPIC state of the VM. Upon VM creation and reset,
vLAPIC state is set to VM_VLAPIC_XAPIC, as ACRN starts the vCPUs vLAPIC in
XAPIC mode.

Tracked-On: #3253
Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

Author: Sainath Grandhi

hypervisor/arch/x86/guest/vlapic.c

@@ -1763,6 +1763,7 @@ int32_t vlapic_set_apicbase(struct acrn_vlapic *vlapic, uint64_t new)
 	int32_t ret = 0;
 	uint64_t changed;
 	bool change_in_vlapic_mode = false;
+	struct acrn_vcpu *vcpu = vlapic->vcpu;
 
 
 	if (vlapic->msr_apicbase != new) {
@@ -1783,6 +1784,7 @@ int32_t vlapic_set_apicbase(struct acrn_vlapic *vlapic, uint64_t new)
 				vlapic->msr_apicbase = new;
 				vlapic_build_x2apic_id(vlapic);
 				switch_apicv_mode_x2apic(vlapic->vcpu);
+				update_vm_vlapic_state(vcpu->vm);
 			} else {
 				/*
 				 * TODO: Logic to check for Invalid transitions, Invalid State

hypervisor/arch/x86/guest/vm.c

@@ -477,6 +477,9 @@ int32_t create_vm(uint16_t vm_id, struct acrn_vm_config *vm_config, struct acrn_
 
 		INIT_LIST_HEAD(&vm->softirq_dev_entry_list);
 		spinlock_init(&vm->softirq_dev_lock);
+		spinlock_init(&vm->vm_lock);
+
+		vm->arch_vm.vlapic_state = VM_VLAPIC_XAPIC;
 		vm->intr_inject_delay_delta = 0UL;
 
 		/* Set up IO bit-mask such that VM exit occurs on
@@ -622,6 +625,11 @@ int32_t reset_vm(struct acrn_vm *vm)
 		foreach_vcpu(i, vm, vcpu) {
 			reset_vcpu(vcpu);
 		}
+		/*
+		 * Set VM vLAPIC state to VM_VLAPIC_XAPIC
+		 */
+
+		vm->arch_vm.vlapic_state = VM_VLAPIC_XAPIC;
 
 		if (is_sos_vm(vm)) {
 			(void )vm_sw_loader(vm);
@@ -632,6 +640,7 @@ int32_t reset_vm(struct acrn_vm *vm)
 		destroy_secure_world(vm, false);
 		vm->sworld_control.flag.active = 0UL;
 		vm->state = VM_CREATED;
+
 		ret = 0;
 	} else {
 		ret = -1;
@@ -759,3 +768,76 @@ void launch_vms(uint16_t pcpu_id)
 		}
 	}
 }
+
+/*
+ * @brief Update state of vLAPICs of a VM
+ * vLAPICs of VM switch between modes in an asynchronous fashion. This API
+ * captures the "transition" state triggered when one vLAPIC switches mode.
+ * When the VM is created, the state is set to "xAPIC" as all vLAPICs are setup
+ * in xAPIC mode.
+ *
+ * Upon reset, all LAPICs switch to xAPIC mode accroding to SDM 10.12.5
+ * Considering VM uses x2apic mode for vLAPIC, in reset or shutdown flow, vLAPIC state
+ * moves to "xAPIC" directly without going thru "transition".
+ *
+ * VM_VLAPIC_X2APIC - All the online vCPUs/vLAPICs of this VM use x2APIC mode
+ * VM_VLAPIC_XAPIC - All the online vCPUs/vLAPICs of this VM use xAPIC mode
+ * VM_VLAPIC_DISABLED - All the online vCPUs/vLAPICs of this VM are in Disabled mode
+ * VM_VLAPIC_TRANSITION - Online vCPUs/vLAPICs of this VM are in between transistion
+ *
+ * TODO: offline_vcpu need to call this API to reflect the status of rest of the
+ * vLAPICs that are online.
+ *
+ * @pre vm != NULL
+ */
+void update_vm_vlapic_state(struct acrn_vm *vm)
+{
+	uint16_t i;
+	struct acrn_vcpu *vcpu;
+	uint16_t vcpus_in_x2apic, vcpus_in_xapic;
+	enum vm_vlapic_state vlapic_state = VM_VLAPIC_XAPIC;
+
+	vcpus_in_x2apic = 0U;
+	vcpus_in_xapic = 0U;
+	spinlock_obtain(&vm->vm_lock);
+	foreach_vcpu(i, vm, vcpu) {
+		if (is_x2apic_enabled(vcpu_vlapic(vcpu))) {
+			vcpus_in_x2apic++;
+		} else if (is_xapic_enabled(vcpu_vlapic(vcpu))) {
+			vcpus_in_xapic++;
+		} else {
+			/*
+			 * vCPU is using vLAPIC in Disabled mode
+			 */
+		}
+	}
+
+	if ((vcpus_in_x2apic == 0U) && (vcpus_in_xapic == 0U)) {
+		/*
+		 * Check if the counts vcpus_in_x2apic and vcpus_in_xapic are zero
+		 * VM_VLAPIC_DISABLED
+		 */
+		vlapic_state = VM_VLAPIC_DISABLED;
+	} else if ((vcpus_in_x2apic != 0U) && (vcpus_in_xapic != 0U)) {
+		/*
+		 * Check if the counts vcpus_in_x2apic and vcpus_in_xapic are non-zero
+		 * VM_VLAPIC_TRANSITION
+		 */
+		vlapic_state = VM_VLAPIC_TRANSITION;
+	} else if (vcpus_in_x2apic != 0U) {
+		/*
+		 * Check if the counts vcpus_in_x2apic is non-zero
+		 * VM_VLAPIC_X2APIC
+		 */
+		vlapic_state = VM_VLAPIC_X2APIC;
+	} else {
+		/*
+		 * Count vcpus_in_xapic is non-zero
+		 * VM_VLAPIC_XAPIC
+		 */
+		vlapic_state = VM_VLAPIC_XAPIC;
+	}
+
+	vm->arch_vm.vlapic_state = vlapic_state;
+	spinlock_release(&vm->vm_lock);
+}

hypervisor/include/arch/x86/guest/vm.h

@@ -76,6 +76,13 @@ enum vm_state {
 	VM_PAUSED,	/* VM paused */
 };
 
+enum vm_vlapic_state {
+	VM_VLAPIC_DISABLED = 0U,
+	VM_VLAPIC_XAPIC,
+	VM_VLAPIC_X2APIC,
+	VM_VLAPIC_TRANSITION
+};
+
 struct vm_arch {
 	/* I/O bitmaps A and B for this VM, MUST be 4-Kbyte aligned */
 	uint8_t io_bitmap[PAGE_SIZE*2];
@@ -93,6 +100,7 @@ struct vm_arch {
 	void *tmp_pg_array;	/* Page array for tmp guest paging struct */
 	struct acrn_vioapic vioapic;	/* Virtual IOAPIC base address */
 	struct acrn_vpic vpic;      /* Virtual PIC */
+	enum vm_vlapic_state vlapic_state; /* Represents vLAPIC state across vCPUs*/
 
 	/* reference to virtual platform to come here (as needed) */
 } __aligned(PAGE_SIZE);
@@ -109,7 +117,7 @@ struct acrn_vm {
 	struct acrn_vuart vuart[MAX_VUART_NUM_PER_VM];		/* Virtual UART */
 	enum vpic_wire_mode wire_mode;
 	struct iommu_domain *iommu;	/* iommu domain of this VM */
-	spinlock_t spinlock;	/* Spin-lock used to protect VM modifications */
+	spinlock_t vm_lock;	/* Spin-lock used to protect VM modifications */
 
 	uint16_t emul_mmio_regions; /* Number of emulated mmio regions */
 	struct mem_io_node emul_mmio[CONFIG_MAX_EMULATED_MMIO_REGIONS];
@@ -218,6 +226,7 @@ bool is_lapic_pt_configured(const struct acrn_vm *vm);
 bool is_rt_vm(const struct acrn_vm *vm);
 bool is_highest_severity_vm(const struct acrn_vm *vm);
 bool vm_hide_mtrr(const struct acrn_vm *vm);
+void update_vm_vlapic_state(struct acrn_vm *vm);
 
 #endif /* !ASSEMBLER */