x86: Jailhouse port, tested for UART (# 0, polling) and LOAPIC timer

This is an introductory port for Zephyr to be run as a Jailhouse
hypervisor[1]'s "inmate cell", on x86 64-bit CPUs (running on 32-bit
mode). This was tested with their "tiny-demo" inmate demo cell
configuration, which takes one of the CPUs of the QEMU-VM root cell
config, along with some RAM and serial controller access (it will even
do nice things like reserving some L3 cache for it via Intel CAT) and
Zephyr samples:

   - hello_world
   - philosophers
   - synchronization

The final binary receives an additional boot sequence preamble that
conforms to Jailhouse's expectations (starts at 0x0 in real mode). It
will put the processor in 32-bit protected mode and then proceed to
Zephyr's __start function.

Testing it is just a matter of:
  $ mmake -C samples/<sample_dir> BOARD=x86_jailhouse JAILHOUSE_QEMU_IMG_FILE=<path_to_image.qcow2> run
  $ sudo insmod <path to jailhouse.ko>
  $ sudo jailhouse enable <path to configs/qemu-x86.cell>
  $ sudo jailhouse cell create <path to configs/tiny-demo.cell>
  $ sudo mount -t 9p -o trans/virtio host /mnt
  $ sudo jailhouse cell load tiny-demo /mnt/zephyr.bin
  $ sudo jailhouse cell start tiny-demo
  $ sudo jailhouse cell destroy tiny-demo
  $ sudo jailhouse disable
  $ sudo rmmod jailhouse

For the hello_world demo case, one should then get QEMU's serial port
output similar to:

"""
Created cell "tiny-demo"
Page pool usage after cell creation: mem 275/1480, remap 65607/131072
Cell "tiny-demo" can be loaded
CPU 3 received SIPI, vector 100
Started cell "tiny-demo"
***** BOOTING ZEPHYR OS v1.9.0 - BUILD: Sep 12 2017 20:03:22 *****
Hello World! x86
"""

Note that the Jailhouse's root cell *has to be started in xAPIC
mode* (kernel command line argument 'nox2apic') in order for this to
work. x2APIC support and its reasoning will come on a separate commit.

As a reminder, the make run target introduced for x86_jailhouse board
involves a root cell image with Jailhouse in it, to be launched and then
partitioned (with >= 2 64-bit CPUs in it).

Inmate cell configs with no JAILHOUSE_CELL_PASSIVE_COMMREG flag
set (e.g. apic-demo one) would need extra code in Zephyr to deal with
cell shutdown command responses from the hypervisor.

You may want to fine tune CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC for your
specific CPU—there is no detection from Zephyr with regard to that.

Other config differences from pristine QEMU defaults worth of mention
are:

   - there is no HPET when running as Jailhouse guest. We use the LOAPIC
     timer, instead
   - there is no PIC_DISABLE, because there is no 8259A PIC when running
     as a Jailhouse guest
   - XIP makes no sense also when running as Jailhouse guest, and both
     PHYS_RAM_ADDR/PHYS_LOAD_ADD are set to zero, what tiny-demo cell
     config is set to

This opens up new possibilities for Zephyr, so that usages beyond just
MCUs come to the table. I see special demand coming from
functional-safety related use cases on industry, automotive, etc.

[1] https://github.com/siemens/jailhouse

Reference to Jailhouse's booting preamble code:

Origin: Jailhouse
License: BSD 2-Clause
URL: https://github.com/siemens/jailhouse
commit: 607251b44397666a3cbbf859d784dccf20aba016
Purpose: Dual-licensing of inmate lib code
Maintained-by: Zephyr

Signed-off-by: Gustavo Lima Chaves <gustavo.lima.chaves@intel.com>

arch/x86/core/crt0.S

@@ -42,6 +42,97 @@
 	GTEXT(_sys_soc_resume_from_deep_sleep)
 #endif
 
+#ifdef CONFIG_JAILHOUSE
+
+#define JAILHOUSE_INMATE_CS32   0x8
+#define JAILHOUSE_INMATE_DS32   0x18
+#define MSR_MTRR_DEF_TYPE       0x000002ff
+#define MTRR_ENABLE             0x00000800
+#define X86_CR0_PE              0x00000001
+#define X86_CR0_WP              0x00010000
+#define X86_CR4_PSE             0x00000010
+
+	.code16
+	.section ".boot", "ax"
+
+	.globl __jh_entry
+__jh_entry:
+	lgdtl %cs:gdt_ptr
+
+	mov %cr0,%eax
+	or $X86_CR0_PE,%al
+	mov %eax,%cr0
+
+	ljmpl $JAILHOUSE_INMATE_CS32,$start32
+
+	.code32
+start32:
+	/*
+	 * Minimal bootstrap into 32-bit mode, just to jump to
+	 * __start
+	 */
+
+	/* 4Mb pages */
+	mov %cr4,%eax
+	or $X86_CR4_PSE,%eax
+	mov %eax,%cr4
+
+	/* Enable write protect and protected mode */
+	mov $(X86_CR0_WP | X86_CR0_PE),%eax
+	mov %eax,%cr0
+
+	movl $MSR_MTRR_DEF_TYPE,%ecx
+	rdmsr
+	or $MTRR_ENABLE,%eax
+	wrmsr
+
+	mov $JAILHOUSE_INMATE_DS32,%eax
+	mov %eax,%ds
+	mov %eax,%es
+	mov %eax,%ss
+
+	ljmp $JAILHOUSE_INMATE_CS32, $__start
+
+	.global loader_gdt
+loader_gdt:
+	.quad 0
+
+	/* Boot entry 1 (selector=0x0): 32-bit code descriptor: DPL0 */
+
+	.word   0xffff		/* limit: xffff */
+	.word   0x0000		/* base : xxxx0000 */
+	.byte   0x00		/* base : xx00xxxx */
+	.byte   0x9b		/* Accessed, Code e/r, Present, DPL0 */
+	.byte   0xcf		/* limit: fxxxx, Page Gra, 32bit */
+	.byte   0x00		/* base : 00xxxxxx */
+
+	/* Boot entry 2 (selector=0x0): 16-bit code descriptor: DPL0 */
+
+	.word   0xffff		/* limit: xffff */
+	.word   0x0000		/* base : xxxx0000 */
+	.byte   0x00		/* base : xx00xxxx */
+	.byte   0x9b		/* Accessed, Code e/r, Present, DPL0 */
+	.byte   0x8f		/* limit: fxxxx, Byte Gra, 16bit */
+	.byte   0x00		/* base : 00xxxxxx */
+
+	/* Boot entry 3 (selector=0x0): Data descriptor: DPL0 */
+
+	.word   0xffff		/* limit: xffff */
+	.word   0x0000		/* base : xxxx0000 */
+	.byte   0x00		/* base : xx00xxxx */
+	.byte   0x93		/* Accessed, Data r/w, Present, DPL0 */
+	.byte   0xcf		/* limit: fxxxx, Page Gra, 32bit */
+	.byte   0x00		/* base : 00xxxxxx */
+gdt_ptr:
+	.short gdt_ptr - loader_gdt - 1
+	.long loader_gdt
+
+	.pushsection ".rodata"
+
+	.align(4096)
+
+	.popsection
+#endif /* CONFIG_JAILHOUSE */
 
 	/* processor is executing in 32-bit protected mode */
 

drivers/interrupt_controller/loapic_intr.c

@@ -227,6 +227,10 @@ static int _loapic_init(struct device *unused)
 
 	/* program Local Vector Table for the Virtual Wire Mode */
 
+	/* skip LINT0/LINT1 for Jailhouse guest case, because we won't
+	 * ever be waiting for interrupts on those
+	 */
+#ifndef CONFIG_JAILHOUSE
 	/* set LINT0: extInt, high-polarity, edge-trigger, not-masked */
 
 	LOAPIC_WRITE(LOAPIC_LINT0, (LOAPIC_READ(LOAPIC_LINT0) &
@@ -240,6 +244,7 @@ static int _loapic_init(struct device *unused)
 		~(LOAPIC_MODE | LOAPIC_LOW |
 		  LOAPIC_LEVEL | LOAPIC_LVT_MASKED)) |
 		(LOAPIC_NMI | LOAPIC_HIGH | LOAPIC_EDGE));
+#endif
 
 	/* lock the Local APIC interrupts */
 

include/arch/x86/linker.ld

@@ -65,10 +65,24 @@
 SECTIONS
 	{
 	GROUP_START(ROMABLE_REGION)
+#ifdef CONFIG_JAILHOUSE
+	/* 16-bit sections */
+	. = PHYS_RAM_ADDR;
+
+	SECTION_PROLOGUE(boot, (OPTIONAL),)
+	{
+	*(.boot)
+	. = ALIGN(16);
+	} GROUP_LINK_IN(ROMABLE_REGION)
+#endif
+	. = ALIGN(8);
 
 	_image_rom_start = PHYS_LOAD_ADDR;
+#ifndef CONFIG_JAILHOUSE
 	_image_text_start = PHYS_LOAD_ADDR;
-
+#else
+	_image_text_start = .;
+#endif
 	SECTION_PROLOGUE(_TEXT_SECTION_NAME, (OPTIONAL),)
 	{
 	. = CONFIG_TEXT_SECTION_OFFSET;
@@ -179,6 +193,16 @@ SECTIONS
 
 	SECTION_PROLOGUE(_BSS_SECTION_NAME, (NOLOAD OPTIONAL),)
 	{
+	/*
+	 * Without Jailhouse, we get the page alignment here for free by
+	 * definition of the beginning of the "RAMable" region on the board
+	 * configurations. With Jailhouse, everything falls in RAM and we
+	 * try to glue sections in sequence, thus we have to realign here so
+	 * that gen_mmu.py does not complain.
+	 */
+#ifdef CONFIG_JAILHOUSE
+	MMU_PAGE_ALIGN
+#endif
 	/*
 	 * For performance, BSS section is forced to be both 4 byte aligned and
 	 * a multiple of 4 bytes.