diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig index c0b42205af1d5..ca4bba377eafe 100644 --- a/arch/x86_64/Kconfig +++ b/arch/x86_64/Kconfig @@ -143,6 +143,14 @@ if ARCH_CHIP_INTEL64_QEMU source "arch/x86_64/src/qemu/Kconfig" endif +config ARCH_X86_64_X2APIC + bool "Use X2APIC mode" + default y + ---help--- + Use X2APIC mode to access local APIC. By default, this option is enabled + because every new CPU supports X2APIC. The main reason for this option is + to support QEMU without the `--enable-kvm` argument. + config ARCH_MULTIBOOT1 bool "Enable ELF32/Multiboot1" default n diff --git a/arch/x86_64/include/intel64/arch.h b/arch/x86_64/include/intel64/arch.h index 0364467ca47b6..e249ecba81344 100644 --- a/arch/x86_64/include/intel64/arch.h +++ b/arch/x86_64/include/intel64/arch.h @@ -276,69 +276,88 @@ # define MSR_IA32_APIC_X2APIC 0x400 # define MSR_IA32_APIC_BSP 0x100 -#define MSR_X2APIC_ID 0x802 -#define MSR_X2APIC_VER 0x803 -#define MSR_X2APIC_TPR 0x808 -#define MSR_X2APIC_PPR 0x80a -#define MSR_X2APIC_EOI 0x80b -#define MSR_X2APIC_LDR 0x80d - -#define MSR_X2APIC_SPIV 0x80f -# define MSR_X2APIC_SPIV_EN 0x100 - -#define MSR_X2APIC_ISR0 0x810 -#define MSR_X2APIC_ISR1 0x811 -#define MSR_X2APIC_ISR2 0x812 -#define MSR_X2APIC_ISR3 0x813 -#define MSR_X2APIC_ISR4 0x814 -#define MSR_X2APIC_ISR5 0x815 -#define MSR_X2APIC_ISR6 0x816 -#define MSR_X2APIC_ISR7 0x817 - -#define MSR_X2APIC_TMR0 0x818 -#define MSR_X2APIC_TMR1 0x819 -#define MSR_X2APIC_TMR2 0x81a -#define MSR_X2APIC_TMR3 0x81b -#define MSR_X2APIC_TMR4 0x81c -#define MSR_X2APIC_TMR5 0x81d -#define MSR_X2APIC_TMR6 0x81e -#define MSR_X2APIC_TMR7 0x81f - -#define MSR_X2APIC_IRR0 0x820 -#define MSR_X2APIC_IRR1 0x821 -#define MSR_X2APIC_IRR2 0x822 -#define MSR_X2APIC_IRR3 0x823 -#define MSR_X2APIC_IRR4 0x824 -#define MSR_X2APIC_IRR5 0x825 -#define MSR_X2APIC_IRR6 0x826 -#define MSR_X2APIC_IRR7 0x827 - -#define MSR_X2APIC_ESR 0x828 -#define MSR_X2APIC_ICR 0x830 -# define MSR_X2APIC_ICR_INIT 0x00000500 /* INIT/RESET */ -# define MSR_X2APIC_ICR_STARTUP 0x00000600 /* Startup IPI */ -# define MSR_X2APIC_ICR_DELIVS 0x00001000 /* Delivery status */ -# define MSR_X2APIC_ICR_ASSERT 0x00004000 /* Assert interrupt (vs deassert) */ -# define MSR_X2APIC_ICR_DEASSERT 0x00000000 -# define MSR_X2APIC_ICR_LEVEL 0x00008000 /* Level triggered */ -# define MSR_X2APIC_ICR_BCAST 0x00080000 /* Send to all APICs, including self. */ -# define MSR_X2APIC_ICR_OTHERS 0x000c0000 /* Send to all APICs, excluding self. */ -# define MSR_X2APIC_ICR_BUSY 0x00001000 -# define MSR_X2APIC_ICR_FIXED 0x00000000 -# define MSR_X2APIC_DESTINATION(d) ((d) << 32) -#define MSR_X2APIC_LVTT 0x832 -# define MSR_X2APIC_LVTT_X1 0x0000000B /* divide counts by 1 */ -# define MSR_X2APIC_LVTT_PERIODIC 0x00020000 /* Periodic */ -# define MSR_X2APIC_LVTT_TSC_DEADLINE 0x00040000 /* Enable TSC DEADLINE One-shot timer */ -#define MSR_X2APIC_LVTTHER 0x833 -#define MSR_X2APIC_LVTPMR 0x834 -#define MSR_X2APIC_LINT0 0x835 -#define MSR_X2APIC_LINT1 0x836 -#define MSR_X2APIC_LERR 0x837 -# define MSR_X2APIC_MASKED 0x00010000 /* Interrupt masked */ -#define MSR_X2APIC_TMICT 0x838 -#define MSR_X2APIC_TMCCT 0x839 -#define MSR_X2APIC_TDCR 0x83e +#define MSR_IA32_PM_ENABLE 0x770 /* Enable/disable HWP (R/W) */ +#define MSR_IA32_HWP_CAPABILITIES 0x771 /* HWP Performance Range Enumeration (R/O) */ +#define MSR_IA32_HWP_REQUEST_PKG 0x772 /* Power Management Control Hints for All Logical Processors in a Package (R/W) */ +#define MSR_IA32_HWP_INTERRUPT 0x773 /* Control HWP Native Interrupts (R/W) */ +#define MSR_IA32_HWP_REQUEST 0x774 /* Power Management Control Hints to a Logical Processor (R/W) */ +#define MSR_IA32_PECI_HWP_REQUEST_INFO 0x775 +#define MSR_IA32_HWP_CTL 0x776 +#define MSR_IA32_HWP_STATUS 0x777 + +/* APIC */ + +#define APIC_BASE 0xfee00000 + +#define APIC_ID 0x0020 +#define APIC_VER 0x0030 +#define APIC_TPR 0x0080 +#define APIC_PPR 0x00a0 +#define APIC_EOI 0x00b0 +# define APIC_EOI_ACK 0 +#define APIC_LDR 0x00d0 +#define APIC_SPIV 0x00f0 +# define APIC_SPIV_EN 0x100 + +#define APIC_ISR0 0x0100 +#define APIC_ISR1 0x0110 +#define APIC_ISR2 0x0120 +#define APIC_ISR3 0x0130 +#define APIC_ISR4 0x0140 +#define APIC_ISR5 0x0150 +#define APIC_ISR6 0x0160 +#define APIC_ISR7 0x0170 + +#define APIC_TMR0 0x0180 +#define APIC_TMR1 0x0190 +#define APIC_TMR2 0x01a0 +#define APIC_TMR3 0x01b0 +#define APIC_TMR4 0x01c0 +#define APIC_TMR5 0x01d0 +#define APIC_TMR6 0x01e0 +#define APIC_TMR7 0x01f0 + +#define APIC_IRR0 0x0200 +#define APIC_IRR1 0x0210 +#define APIC_IRR2 0x0220 +#define APIC_IRR3 0x0230 +#define APIC_IRR4 0x0240 +#define APIC_IRR5 0x0250 +#define APIC_IRR6 0x0260 +#define APIC_IRR7 0x0270 + +#define APIC_ESR 0x0280 +#define APIC_ICR 0x0300 +# define APIC_ICR_INIT 0x00000500 /* INIT/RESET */ +# define APIC_ICR_STARTUP 0x00000600 /* Startup IPI */ +# define APIC_ICR_DELIVS 0x00001000 /* Delivery status */ +# define APIC_ICR_ASSERT 0x00004000 /* Assert interrupt (vs deassert) */ +# define APIC_ICR_DEASSERT 0x00000000 +# define APIC_ICR_LEVEL 0x00008000 /* Level triggered */ +# define APIC_ICR_BCAST 0x00080000 /* Send to all APICs, including self. */ +# define APIC_ICR_OTHERS 0x000c0000 /* Send to all APICs, excluding self. */ +# define APIC_ICR_BUSY 0x00001000 +# define APIC_ICR_FIXED 0x00000000 +# define X2APIC_DESTINATION(d) ((d) << 32) +# define APIC_DESTINATION(d) ((d) << 24) +#define APIC_ICR_HIGH 0x0310 +#define APIC_LVTT 0x0320 +# define APIC_LVTT_X1 0x0000000B /* divide counts by 1 */ +# define APIC_LVTT_PERIODIC 0x00020000 /* Periodic */ +# define APIC_LVTT_TSC_DEADLINE 0x00040000 /* Enable TSC DEADLINE One-shot timer */ +#define APIC_LVTTHER 0x0330 +#define APIC_LVTPMR 0x0340 +#define APIC_LINT0 0x0350 +#define APIC_LINT1 0x0360 +#define APIC_LERR 0x0370 +# define APIC_MASKED 0x00010000 /* Interrupt masked */ +#define APIC_TMICT 0x0380 +#define APIC_TMCCT 0x0390 +#define APIC_TDCR 0x03e0 + +/* MSR */ + +#define MSR_X2APIC_BASE 0x800 #define MSR_IA32_XSS 0xda0 /* IOAPIC related Definitions */ diff --git a/arch/x86_64/include/intel64/irq.h b/arch/x86_64/include/intel64/irq.h index 84a04a4436fef..fb93f975fd8e1 100644 --- a/arch/x86_64/include/intel64/irq.h +++ b/arch/x86_64/include/intel64/irq.h @@ -862,9 +862,36 @@ static inline_function void up_irq_restore(irqstate_t flags) } } +static inline_function void apic_write(unsigned int offset, + unsigned long val) +{ +#ifdef CONFIG_ARCH_X86_64_X2APIC + write_msr((offset >> 4) + MSR_X2APIC_BASE, val); +#else + uintptr_t addr = APIC_BASE + offset; + + /* xAPIC access must be 32-bit aligned */ + + *((FAR volatile uint32_t *)addr) = val; +#endif +} + +static inline_function unsigned long apic_read(unsigned int offset) +{ +#ifdef CONFIG_ARCH_X86_64_X2APIC + return read_msr((offset >> 4) + MSR_X2APIC_BASE); +#else + uintptr_t addr = APIC_BASE + offset; + + /* xAPIC access must be 32-bit aligned */ + + return *((FAR volatile uint32_t *)addr); +#endif +} + static inline_function unsigned int up_apic_cpu_id(void) { - return read_msr(MSR_X2APIC_ID); + return apic_read(APIC_ID); } /**************************************************************************** diff --git a/arch/x86_64/src/common/x86_64_internal.h b/arch/x86_64/src/common/x86_64_internal.h index 3c787352eecb4..a81e0324cb110 100644 --- a/arch/x86_64/src/common/x86_64_internal.h +++ b/arch/x86_64/src/common/x86_64_internal.h @@ -308,6 +308,10 @@ void x86_64_stack_color(void *stackbase, size_t nbytes); int x86_64_tlb_handler(int irq, void *c, void *arg); void x86_64_tlb_shootdown(void); +/* Defined in intel64_irq.c */ + +void x86_64_icr_write(uint8_t cpu, uint32_t delivery); + #endif /* __ASSEMBLY__ */ #endif /* __ARCH_X86_64_SRC_COMMON_UP_INTERNAL_H */ diff --git a/arch/x86_64/src/intel64/intel64_check_capability.c b/arch/x86_64/src/intel64/intel64_check_capability.c index ca642297563c4..2bebe3cdac142 100644 --- a/arch/x86_64/src/intel64/intel64_check_capability.c +++ b/arch/x86_64/src/intel64/intel64_check_capability.c @@ -98,9 +98,11 @@ void x86_64_check_and_enable_capability(void) require |= X86_64_CPUID_01_SSE42; #endif +#ifdef CONFIG_ARCH_X86_64_X2APIC /* Check x2APIC availability */ require |= X86_64_CPUID_01_X2APIC; +#endif /* Check timer availability */ diff --git a/arch/x86_64/src/intel64/intel64_cpustart.c b/arch/x86_64/src/intel64/intel64_cpustart.c index fe19a5439f703..d6e43a4faf712 100644 --- a/arch/x86_64/src/intel64/intel64_cpustart.c +++ b/arch/x86_64/src/intel64/intel64_cpustart.c @@ -71,25 +71,17 @@ extern uint64_t get_tsc_adjust(void); static int x86_64_ap_startup(int cpu) { - uint64_t dest = 0; - uint64_t vect = 0; - uint64_t regval = 0; + uint64_t vect = 0; sinfo("cpu=%d\n", cpu); - /* Get destination - must be LOAPIC id */ - - dest = MSR_X2APIC_DESTINATION((uint64_t)x86_64_cpu_to_loapic(cpu)); - /* Get the AP trampoline from a fixed address */ vect = (uint32_t)((uintptr_t)&__ap_entry) >> 12; /* Send an INIT IPI to the CPU */ - regval = MSR_X2APIC_ICR_INIT | MSR_X2APIC_ICR_ASSERT - | MSR_X2APIC_ICR_LEVEL | dest; - write_msr(MSR_X2APIC_ICR, regval); + x86_64_icr_write(cpu, APIC_ICR_INIT | APIC_ICR_ASSERT | APIC_ICR_LEVEL); /* Wait for 10 ms */ @@ -98,8 +90,7 @@ static int x86_64_ap_startup(int cpu) /* Send an STARTUP IPI to the CPU */ - regval = MSR_X2APIC_ICR_STARTUP | dest | vect; - write_msr(MSR_X2APIC_ICR, regval); + x86_64_icr_write(cpu, APIC_ICR_STARTUP | vect); /* Wait for AP ready */ diff --git a/arch/x86_64/src/intel64/intel64_handlers.c b/arch/x86_64/src/intel64/intel64_handlers.c index 6835f19667053..295d5f6a504a0 100644 --- a/arch/x86_64/src/intel64/intel64_handlers.c +++ b/arch/x86_64/src/intel64/intel64_handlers.c @@ -41,14 +41,6 @@ #include "x86_64_internal.h" #include "sched/sched.h" -/**************************************************************************** - * Pre-processor Definitions - ****************************************************************************/ - -#define X2APIC_EOI 0x80b - -#define APIC_EOI_ACK 0 - /**************************************************************************** * Private Functions ****************************************************************************/ @@ -164,7 +156,7 @@ uint64_t *irq_handler(uint64_t *regs, uint64_t irq_no) /* Send an EOI (end of interrupt) signal to the APIC */ - write_msr(X2APIC_EOI, APIC_EOI_ACK); + apic_write(APIC_EOI, APIC_EOI_ACK); board_autoled_off(LED_INIRQ); return ret; #endif diff --git a/arch/x86_64/src/intel64/intel64_irq.c b/arch/x86_64/src/intel64/intel64_irq.c index dc6c1b465db0a..7bf1d4163b366 100644 --- a/arch/x86_64/src/intel64/intel64_irq.c +++ b/arch/x86_64/src/intel64/intel64_irq.c @@ -211,7 +211,39 @@ static void up_deinit_8259(void) outb(X86_PIC_EOI, X86_IO_PORT_PIC1_CMD); outb(X86_PIC_EOI, X86_IO_PORT_PIC2_CMD); } + +/**************************************************************************** + * Name: up_apic_enable + * + * Description: + * Enable xAPIC/x2APCI. + * + ****************************************************************************/ + +static void up_apic_enable(void) +{ + uint32_t apic_base; + + apic_base = read_msr(MSR_IA32_APIC_BASE) & 0xfffff000; + +#ifdef CONFIG_ARCH_X86_64_X2APIC + /* Enable the APIC in X2APIC MODE */ + + write_msr(MSR_IA32_APIC_BASE, apic_base | MSR_IA32_APIC_EN | + MSR_IA32_APIC_X2APIC | MSR_IA32_APIC_BSP); +#else + /* Check xAPIC base - for now it must be 0xfee00000 */ + + ASSERT(apic_base == APIC_BASE); + + /* Map xAPIC memory region */ + + up_map_region((void *)APIC_BASE, 0x1000, + X86_PAGE_WR | X86_PAGE_PRESENT | + X86_PAGE_NOCACHE | X86_PAGE_GLOBAL); #endif +} +#endif /* CONFIG_ARCH_INTEL64_DISABLE_INT_INIT */ /**************************************************************************** * Name: up_init_apic @@ -225,62 +257,59 @@ static void up_apic_init(void) { uint32_t ver; uint32_t icrl; - uint32_t apic_base; -#ifndef CONFIG_ARCH_INTEL64_DISABLE_INT_INIT - /* Enable the APIC in X2APIC MODE */ +#if !defined(CONFIG_ARCH_INTEL64_DISABLE_INT_INIT) + /* Enable xAPIC or x2APIC */ - apic_base = read_msr(MSR_IA32_APIC_BASE) & 0xfffff000; - write_msr(MSR_IA32_APIC_BASE, apic_base | MSR_IA32_APIC_EN | - MSR_IA32_APIC_X2APIC | MSR_IA32_APIC_BSP); + up_apic_enable(); #endif /* Enable the APIC and setup an spurious interrupt vector */ - write_msr(MSR_X2APIC_SPIV, MSR_X2APIC_SPIV_EN | IRQ_SPURIOUS); + apic_write(APIC_SPIV, APIC_SPIV_EN | IRQ_SPURIOUS); #ifndef CONFIG_ARCH_INTEL64_DISABLE_INT_INIT /* Disable the LINT interrupt lines */ - write_msr(MSR_X2APIC_LINT0, MSR_X2APIC_MASKED); - write_msr(MSR_X2APIC_LINT1, MSR_X2APIC_MASKED); + apic_write(APIC_LINT0, APIC_MASKED); + apic_write(APIC_LINT1, APIC_MASKED); /* Disable performance counter overflow interrupts on machines which * provide that interrupt entry. */ - ver = read_msr(MSR_X2APIC_VER); + ver = apic_read(APIC_VER); if (((ver >> 16) & 0xff) >= 4) { - write_msr(MSR_X2APIC_LVTPMR, MSR_X2APIC_MASKED); + apic_write(APIC_LVTPMR, APIC_MASKED); } /* Map error interrupt to IRQ_ERROR. */ - write_msr(MSR_X2APIC_LERR, MSR_X2APIC_MASKED); + apic_write(APIC_LERR, APIC_MASKED); /* Clear error status register (requires back-to-back writes). */ - write_msr(MSR_X2APIC_ESR, 0); - write_msr(MSR_X2APIC_ESR, 0); + apic_write(APIC_ESR, 0); + apic_write(APIC_ESR, 0); /* Ack any outstanding interrupts. */ - write_msr(MSR_X2APIC_EOI, 0); + apic_write(APIC_EOI, 0); /* Send an Init Level De-Assert to synchronize arbitration ID's. */ - write_msr(MSR_X2APIC_ICR, MSR_X2APIC_ICR_BCAST | MSR_X2APIC_ICR_INIT | - MSR_X2APIC_ICR_LEVEL); + apic_write(APIC_ICR, APIC_ICR_BCAST | APIC_ICR_INIT | + APIC_ICR_LEVEL); do { - icrl = read_msr(MSR_X2APIC_ICR); + icrl = apic_read(APIC_ICR); } - while (icrl & MSR_X2APIC_ICR_DELIVS); + while (icrl & APIC_ICR_DELIVS); /* Enable interrupts on the APIC (but not on the processor). */ - write_msr(MSR_X2APIC_TPR, 0); + apic_write(APIC_TPR, 0); #endif } @@ -674,12 +703,7 @@ void up_trigger_irq(int irq, cpu_set_t cpuset) { if (CPU_ISSET(cpu, &cpuset)) { - write_msr(MSR_X2APIC_ICR, - MSR_X2APIC_ICR_FIXED | - MSR_X2APIC_ICR_ASSERT | - MSR_X2APIC_DESTINATION( - (uint64_t)x86_64_cpu_to_loapic(cpu)) | - irq); + x86_64_icr_write(cpu, APIC_ICR_FIXED | APIC_ICR_ASSERT | irq); } } } @@ -883,3 +907,29 @@ void up_affinity_irq(int irq, cpu_set_t cpuset) spin_unlock_irqrestore(&g_irq_spinlock, flags); } + +/**************************************************************************** + * Name: x86_64_icr_write + * + * Description: + * Write ICR request + * + ****************************************************************************/ + +void x86_64_icr_write(uint8_t cpu, uint32_t delivery) +{ +#ifdef CONFIG_ARCH_X86_64_X2APIC + /* Get destination - must be LOAPIC id */ + + apic_write(APIC_ICR, delivery | + X2APIC_DESTINATION((uint64_t)x86_64_cpu_to_loapic(cpu))); +#else + /* ICR high */ + + apic_write(APIC_ICR_HIGH, APIC_DESTINATION(x86_64_cpu_to_loapic(cpu))); + + /* ICR low */ + + apic_write(APIC_ICR, delivery); +#endif +} diff --git a/arch/x86_64/src/intel64/intel64_tsc_oneshot.c b/arch/x86_64/src/intel64/intel64_tsc_oneshot.c index d93a0cf64ca91..fc8b487a34386 100644 --- a/arch/x86_64/src/intel64/intel64_tsc_oneshot.c +++ b/arch/x86_64/src/intel64/intel64_tsc_oneshot.c @@ -106,10 +106,10 @@ static inline_function void intel64_mask_tmr(void) /* Disable TSC Deadline interrupt */ #ifdef CONFIG_ARCH_INTEL64_TSC_DEADLINE - write_msr(MSR_X2APIC_LVTT, TMR_IRQ | MSR_X2APIC_LVTT_TSC_DEADLINE | - (1 << 16)); + apic_write(APIC_LVTT, TMR_IRQ | APIC_LVTT_TSC_DEADLINE | + (1 << 16)); #else - write_msr(MSR_X2APIC_LVTT, TMR_IRQ | (1 << 16)); + apic_write(APIC_LVTT, TMR_IRQ | (1 << 16)); #endif /* Required when using TSC deadline mode. */ @@ -122,9 +122,9 @@ static inline_function void intel64_unmask_tmr(void) /* Enable TSC Deadline interrupt */ #ifdef CONFIG_ARCH_INTEL64_TSC_DEADLINE - write_msr(MSR_X2APIC_LVTT, TMR_IRQ | MSR_X2APIC_LVTT_TSC_DEADLINE); + apic_write(APIC_LVTT, TMR_IRQ | APIC_LVTT_TSC_DEADLINE); #else - write_msr(MSR_X2APIC_LVTT, TMR_IRQ); + apic_write(APIC_LVTT, TMR_IRQ); #endif /* Required when using TSC deadline mode. */ @@ -213,6 +213,7 @@ static void intel64_tsc_start(struct oneshot_lowerhalf_s *lower, clkcnt_t now = intel64_tsc_count() + intel64_tsc_get_offset(); clkcnt_t expected = now + delta >= now ? now + delta : UINT64_MAX; + intel64_tsc_set_compare(expected); } diff --git a/arch/x86_64/src/intel64/intel64_tsc_timerisr.c b/arch/x86_64/src/intel64/intel64_tsc_timerisr.c index f097032d58872..e466e371dea4f 100644 --- a/arch/x86_64/src/intel64/intel64_tsc_timerisr.c +++ b/arch/x86_64/src/intel64/intel64_tsc_timerisr.c @@ -75,9 +75,9 @@ void apic_timer_set(unsigned long timeout_ns) (unsigned long long)timeout_ns * g_x86_64_timer_freq / NS_PER_SEC; #ifdef CONFIG_ARCH_INTEL64_TSC_DEADLINE - write_msr(MSR_IA32_TSC_DEADLINE, rdtscp() + ticks); + write_msr(MSR_IA32_TSC_DEADLINE, rdtscp() + ticks); #else - write_msr(MSR_X2APIC_TMICT, ticks); + apic_write(APIC_TMICT, ticks); #endif } @@ -119,10 +119,10 @@ void up_timer_initialize(void) irq_attach(IRQ0, (xcpt_t)intel64_timerisr, NULL); #ifdef CONFIG_ARCH_INTEL64_TSC_DEADLINE - vector |= MSR_X2APIC_LVTT_TSC_DEADLINE; + vector |= APIC_LVTT_TSC_DEADLINE; #endif - write_msr(MSR_X2APIC_LVTT, vector); + apic_write(APIC_LVTT, vector); __asm__ volatile("mfence" : : : "memory");