jailhouse: add support for x2APIC mode for all LOAPIC accesses

Besides the fact that we did not have that for the current supported
boards, that makes sense for this new, virtualized mode, that is meant
to be run on top of full-fledged x86 64 CPUs.

By having xAPIC mode access only, Jailhouse has to intercept those MMIO
reads and writes, in order to examine what they do and arbitrate if it's
safe or not (e.g. not all values are accepted to ICR register). This
means that we can't run away from having a VM-exit event for each and
every access to APIC memory region and this impacts the latency the
guest OS observes over bare metal a lot.

When in x2APIC mode, Jailhouse does not require VM-exits for MSR
accesses other that writes to the ICR register, so the latency the guest
observes is reduced to almost zero.

Here are some outputs of the the command line

  $ sudo ./tools/jailhouse cell stats tiny-demo

on a Jailhouse's root cell console, for one of the Zephyr demos using
LOAPIC timers, left for a couple of seconds:

Statistics for tiny-demo cell (x2APIC root, x2APIC inmate)

COUNTER                              SUM   PER SEC
vmexits_total                          7         0
vmexits_management                     3         0
vmexits_cr                             2         0
vmexits_cpuid                          1         0
vmexits_msr                            1         0
vmexits_exception                      0         0
vmexits_hypercall                      0         0
vmexits_mmio                           0         0
vmexits_pio                            0         0
vmexits_xapic                          0         0
vmexits_xsetbv                         0         0

Statistics for tiny-demo cell (xAPIC root, xAPIC inmate)

COUNTER                              SUM   PER SEC
vmexits_total                       4087        40
vmexits_xapic                       4080        40
vmexits_management                     3         0
vmexits_cr                             2         0
vmexits_cpuid                          1         0
vmexits_msr                            1         0
vmexits_exception                      0         0
vmexits_hypercall                      0         0
vmexits_mmio                           0         0
vmexits_pio                            0         0
vmexits_xsetbv                         0         0

Statistics for tiny-demo cell (xAPIC root, x2APIC inmate)

COUNTER                              SUM   PER SEC
vmexits_total                       4087        40
vmexits_msr                         4080        40
vmexits_management                     3         0
vmexits_cr                             2         0
vmexits_cpuid                          1         0
vmexits_exception                      0         0
vmexits_hypercall                      0         0
vmexits_mmio                           0         0
vmexits_pio                            0         0
vmexits_xapic                          0         0
vmexits_xsetbv                         0         0

See that under x2APIC mode on both Jailhouse/root-cell and guest, the
interruptions from the hypervisor are minimal. That is not the case when
Jailhouse is on xAPIC mode, though. Note also that, as a plus, x2APIC
accesses on the guest will map to xAPIC MMIO on the hypervisor just
fine.

Signed-off-by: Gustavo Lima Chaves <gustavo.lima.chaves@intel.com>
This commit is contained in:
Gustavo Lima Chaves 2017-10-11 14:17:12 -07:00 committed by Anas Nashif
commit 9bb07ff69a
4 changed files with 60 additions and 0 deletions

View file

@ -69,6 +69,20 @@ extern void k_cpu_atomic_idle(unsigned int imask);
extern void _MsrWrite(unsigned int msr, u64_t msrData);
extern u64_t _MsrRead(unsigned int msr);
#ifdef CONFIG_JAILHOUSE_X2APIC
#define MSR_X2APIC_BASE 0x00000800
static inline u32_t read_x2apic(unsigned int reg)
{
return _MsrRead(MSR_X2APIC_BASE + reg);
}
static inline void write_x2apic(unsigned int reg, u32_t val)
{
_MsrWrite(MSR_X2APIC_BASE + reg, val);
}
#endif
/*
* _IntLibInit() is called from the non-arch specific function,
* prepare_multithreading(). The IA-32 kernel does not require any special

View file

@ -78,6 +78,7 @@
*/
#include <kernel.h>
#include <kernel_structs.h>
#include <arch/cpu.h>
#include <zephyr/types.h>
#include <string.h>
@ -190,12 +191,20 @@ static u32_t loapic_device_power_state = DEVICE_PM_ACTIVE_STATE;
static ALWAYS_INLINE u32_t LOAPIC_READ(mem_addr_t addr)
{
#ifndef CONFIG_JAILHOUSE_X2APIC
return sys_read32(CONFIG_LOAPIC_BASE_ADDRESS + addr);
#else
return read_x2apic(addr >> 4);
#endif
}
static ALWAYS_INLINE void LOAPIC_WRITE(mem_addr_t addr, u32_t data)
{
#ifndef CONFIG_JAILHOUSE_X2APIC
sys_write32(data, CONFIG_LOAPIC_BASE_ADDRESS + addr);
#else
write_x2apic(addr >> 4, data);
#endif
}
/**
@ -219,7 +228,10 @@ static int _loapic_init(struct device *unused)
/* reset the DFR, TPR, TIMER_CONFIG, and TIMER_ICR */
/* Jailhouse does not allow writes to DFR in x2APIC mode */
#ifndef CONFIG_JAILHOUSE_X2APIC
LOAPIC_WRITE(LOAPIC_DFR, 0xffffffff);
#endif
LOAPIC_WRITE(LOAPIC_TPR, 0x0);
LOAPIC_WRITE(LOAPIC_TIMER_CONFIG, 0x0);

View file

@ -150,6 +150,13 @@ static u32_t reg_timer_cfg_save;
#endif
#endif
#ifdef CONFIG_JAILHOUSE_X2APIC
void _jailhouse_eoi(void)
{
write_x2apic(LOAPIC_EOI >> 4, 0);
}
#endif
/**
*
* @brief Set the timer for periodic mode
@ -160,7 +167,12 @@ static u32_t reg_timer_cfg_save;
*/
static inline void periodic_mode_set(void)
{
#ifndef CONFIG_JAILHOUSE_X2APIC
*_REG_TIMER |= LOAPIC_TIMER_PERIODIC;
#else
write_x2apic(LOAPIC_TIMER >> 4,
read_x2apic(LOAPIC_TIMER >> 4) | LOAPIC_TIMER_PERIODIC);
#endif
}
@ -176,7 +188,11 @@ static inline void periodic_mode_set(void)
*/
static inline void initial_count_register_set(u32_t count)
{
#ifndef CONFIG_JAILHOUSE_X2APIC
*_REG_TIMER_ICR = count;
#else
write_x2apic(LOAPIC_TIMER_ICR >> 4, count);
#endif
}
#if defined(CONFIG_TICKLESS_IDLE)
@ -207,7 +223,13 @@ static inline void one_shot_mode_set(void)
#ifndef CONFIG_MVIC
static inline void divide_configuration_register_set(void)
{
#ifndef CONFIG_JAILHOUSE_X2APIC
*_REG_TIMER_CFG = (*_REG_TIMER_CFG & ~0xf) | LOAPIC_TIMER_DIVBY_1;
#else
write_x2apic(LOAPIC_TIMER_CONFIG >> 4,
(read_x2apic(LOAPIC_TIMER_CONFIG >> 4) & ~0xf)
| LOAPIC_TIMER_DIVBY_1);
#endif
}
#endif
@ -223,7 +245,11 @@ static inline void divide_configuration_register_set(void)
*/
static inline u32_t current_count_register_get(void)
{
#ifndef CONFIG_JAILHOUSE_X2APIC
return *_REG_TIMER_CCR;
#else
return read_x2apic(LOAPIC_TIMER_CCR >> 4);
#endif
}
#if defined(CONFIG_TICKLESS_IDLE)

View file

@ -27,6 +27,10 @@ void __irq_controller_irq_config(unsigned int vector, unsigned int irq,
int __irq_controller_isr_vector_get(void);
#ifdef CONFIG_JAILHOUSE_X2APIC
void _jailhouse_eoi(void);
#endif
static inline void __irq_controller_eoi(void)
{
#if CONFIG_EOI_FORWARDING_BUG
@ -44,9 +48,13 @@ static inline void __irq_controller_eoi(void)
.endm
#else
.macro __irq_controller_eoi_macro
#ifdef CONFIG_JAILHOUSE_X2APIC
call _jailhouse_eoi
#else
xorl %eax, %eax /* zeroes eax */
loapic_eoi_reg = (CONFIG_LOAPIC_BASE_ADDRESS + LOAPIC_EOI)
movl %eax, loapic_eoi_reg /* tell LOAPIC the IRQ is handled */
#endif
.endm
#endif /* CONFIG_EOI_FORWARDING_BUG */