From 3eb1a8b59a641666287503ccd9735fb3a1b981d9 Mon Sep 17 00:00:00 2001 From: "Charles E. Youse" Date: Sat, 28 Sep 2019 22:38:03 -0400 Subject: [PATCH] arch/x86: (Intel64) implement SMP support Add duplicate per-CPU data structures (x86_cpuboot, tss, stacks, etc.) for up to 4 total CPUs, add code in locore and z_arch_start_cpu(). The test board, qemu_x86_long, now defaults to 2 CPUs. Signed-off-by: Charles E. Youse --- arch/x86/core/intel64/cpu.c | 86 +++++++++++- arch/x86/core/intel64/locore.S | 139 +++++++++++++++++++- arch/x86/include/intel64/kernel_arch_data.h | 8 ++ arch/x86/include/kernel_arch_func.h | 3 + boards/x86/qemu_x86/qemu_x86_long_defconfig | 1 + soc/x86/apollo_lake/CMakeLists.txt | 1 + soc/x86/apollo_lake/cpu.c | 8 ++ 7 files changed, 239 insertions(+), 7 deletions(-) create mode 100644 soc/x86/apollo_lake/cpu.c diff --git a/arch/x86/core/intel64/cpu.c b/arch/x86/core/intel64/cpu.c index 751373c2143..f8f8c8a0482 100644 --- a/arch/x86/core/intel64/cpu.c +++ b/arch/x86/core/intel64/cpu.c @@ -8,24 +8,106 @@ #include #include #include +#include + +/* + * Map of CPU logical IDs to CPU local APIC IDs. By default, + * we assume this simple identity mapping, as found in QEMU. + * The symbol is weak so that boards/SoC files can override. + */ + +__weak u8_t x86_cpu_loapics[] = { 0, 1, 2, 3 }; extern FUNC_NORETURN void z_x86_prep_c(int, struct multiboot_info *); +extern char x86_ap_start[]; /* AP entry point in locore.S */ + extern u8_t _exception_stack[]; +extern u8_t _exception_stack1[]; +extern u8_t _exception_stack2[]; +extern u8_t _exception_stack3[]; Z_GENERIC_SECTION(.tss) struct x86_tss64 tss0 = { .ist1 = (u64_t) _interrupt_stack + CONFIG_ISR_STACK_SIZE, .ist7 = (u64_t) _exception_stack + CONFIG_EXCEPTION_STACK_SIZE, - .iomapb = 0xFFFF, /* no I/O access bitmap */ + .iomapb = 0xFFFF, .cpu = &(_kernel.cpus[0]) }; +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 1) +Z_GENERIC_SECTION(.tss) +struct x86_tss64 tss1 = { + .ist1 = (u64_t) _interrupt_stack1 + CONFIG_ISR_STACK_SIZE, + .ist7 = (u64_t) _exception_stack1 + CONFIG_EXCEPTION_STACK_SIZE, + .iomapb = 0xFFFF, + .cpu = &(_kernel.cpus[1]) +}; +#endif + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 2) +Z_GENERIC_SECTION(.tss) +struct x86_tss64 tss2 = { + .ist1 = (u64_t) _interrupt_stack2 + CONFIG_ISR_STACK_SIZE, + .ist7 = (u64_t) _exception_stack2 + CONFIG_EXCEPTION_STACK_SIZE, + .iomapb = 0xFFFF, + .cpu = &(_kernel.cpus[2]) +}; +#endif + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 3) +Z_GENERIC_SECTION(.tss) +struct x86_tss64 tss3 = { + .ist1 = (u64_t) _interrupt_stack3 + CONFIG_ISR_STACK_SIZE, + .ist7 = (u64_t) _exception_stack3 + CONFIG_EXCEPTION_STACK_SIZE, + .iomapb = 0xFFFF, + .cpu = &(_kernel.cpus[3]) +}; +#endif + struct x86_cpuboot x86_cpuboot[] = { { .tr = X86_KERNEL_CPU0_TR, .gs = X86_KERNEL_CPU0_GS, .sp = (u64_t) _interrupt_stack + CONFIG_ISR_STACK_SIZE, .fn = z_x86_prep_c - } + }, +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 1) + { + .tr = X86_KERNEL_CPU1_TR, + .gs = X86_KERNEL_CPU1_GS, + }, +#endif +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 2) + { + .tr = X86_KERNEL_CPU2_TR, + .gs = X86_KERNEL_CPU2_GS, + }, +#endif +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 3) + { + .tr = X86_KERNEL_CPU3_TR, + .gs = X86_KERNEL_CPU3_GS, + }, +#endif }; + +/* + * Send the INIT/STARTUP IPI sequence required to start up CPU 'cpu_num', which + * will enter the kernel at fn(---, arg), running on the specified stack. + */ + +void z_arch_start_cpu(int cpu_num, k_thread_stack_t *stack, int sz, + void (*fn)(int key, void *data), void *arg) +{ + u8_t vector = ((unsigned long) x86_ap_start) >> 12; + u8_t apic_id = x86_cpu_loapics[cpu_num]; + + x86_cpuboot[cpu_num].sp = (u64_t) Z_THREAD_STACK_BUFFER(stack) + sz; + x86_cpuboot[cpu_num].fn = fn; + x86_cpuboot[cpu_num].arg = arg; + + z_loapic_ipi(apic_id, LOAPIC_ICR_IPI_INIT, 0); + k_busy_wait(10000); + z_loapic_ipi(apic_id, LOAPIC_ICR_IPI_STARTUP, vector); +} diff --git a/arch/x86/core/intel64/locore.S b/arch/x86/core/intel64/locore.S index 58bf6b031b6..19eab541e74 100644 --- a/arch/x86/core/intel64/locore.S +++ b/arch/x86/core/intel64/locore.S @@ -14,6 +14,60 @@ .section .locore,"ax" .code32 +#ifdef CONFIG_SMP && (CONFIG_MP_NUM_CPUS > 1) + + /* + * APs are sent here on startup, in real mode. This + * is first because we want it on a page boundary. + */ + +.code16 +.global x86_ap_start +x86_ap_start: + + /* + * First, we move to 32-bit protected mode, and set up the + * same flat environment that the BSP gets from the loader. + */ + + lgdt gdt48 + lidt idt48 + movw $1, %ax + lmsw %ax + jmpl $X86_KERNEL_CS_32, $1f +.code32 +1: movw $X86_KERNEL_DS_32, %ax + movw %ax, %ds + movw %ax, %es + movw %ax, %ss + movw %ax, %fs + movw %ax, %gs + + /* + * Now, reverse-map our local APIC ID to our logical CPU ID + * so we can locate our x86_cpuboot[] bundle. Put it in EBP. + */ + + movl CONFIG_LOAPIC_BASE_ADDRESS+LOAPIC_ID, %eax + shrl $24, %eax + andl $0x0F, %eax /* local APIC ID -> EAX */ + + movl $x86_cpuboot, %ebp + xorl %ebx, %ebx +1: cmpl $CONFIG_MP_NUM_CPUS, %ebx + jz unknown_loapic_id + cmpb %al, x86_cpu_loapics(%ebx) + je go64 /* proceed to 64-bit mode */ + incl %ebx + addl $__X86_CPUBOOT_SIZEOF, %ebp + jmp 1b + +unknown_loapic_id: + jmp unknown_loapic_id + +#endif /* CONFIG_SMP && ... */ + +.code32 .globl __start __start: @@ -23,6 +77,9 @@ __start: * first, let common code do things like detect multiboot info. */ + lgdt gdt48 + lidt idt48 + #include "../common.S" /* @@ -47,7 +104,7 @@ __start: * and configure per-CPU stuff: GS, task register, stack. */ - movl %cr4, %eax /* enable PAE and SSE */ +go64: movl %cr4, %eax /* enable PAE and SSE */ orl $(CR4_PAE | CR4_OSFXSR), %eax movl %eax, %cr4 clts @@ -64,8 +121,6 @@ __start: orl $CR0_PG, %eax movl %eax, %cr0 - lgdt gdt48 - lidt idt48 jmpl $X86_KERNEL_CS, $1f .code64 1: movl $X86_KERNEL_DS, %eax @@ -207,6 +262,10 @@ gdt: .word 0, 0, 0, 0 /* 0x28: unused */ + /* + * CPU 0 task state segment descriptors + */ + .word 0 /* 0x30: 64-bit TSS data (for GS) */ .word tss0 .word 0x9200 @@ -217,11 +276,61 @@ gdt: .word __X86_TSS64_SIZEOF-1 /* 0x40: 64-bit TSS (16-byte entry) */ .word tss0 .word 0x8900 + .word 0, 0, 0, 0, 0 + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 1) + /* + * CPU 1 task state segment descriptors + */ + + .word 0 /* 0x50: 64-bit TSS data (for GS) */ + .word tss1 + .word 0x9200 .word 0 + + .word 0, 0, 0, 0 /* 0x58: unused */ + + .word __X86_TSS64_SIZEOF-1 /* 0x60: 64-bit TSS (16-byte entry) */ + .word tss1 + .word 0x8900 + .word 0, 0, 0, 0, 0 +#endif + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 2) + /* + * CPU 2 task state segment descriptors + */ + + .word 0 /* 0x70: 64-bit TSS data (for GS) */ + .word tss2 + .word 0x9200 .word 0 + + .word 0, 0, 0, 0 /* 0x78: unused */ + + .word __X86_TSS64_SIZEOF-1 /* 0x80: 64-bit TSS (16-byte entry) */ + .word tss2 + .word 0x8900 + .word 0, 0, 0, 0, 0 +#endif + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 3) + /* + * CPU 3 task state segment descriptors + */ + + .word 0 /* 0x90: 64-bit TSS data (for GS) */ + .word tss3 + .word 0x9200 .word 0 - .word 0 - .word 0 + + .word 0, 0, 0, 0 /* 0x98: unused */ + + .word __X86_TSS64_SIZEOF-1 /* 0xA0: 64-bit TSS (16-byte entry) */ + .word tss3 + .word 0x8900 + .word 0, 0, 0, 0, 0 +#endif gdt48: .word (gdt48 - gdt - 1) @@ -542,3 +651,23 @@ pdp: .long 0x00000183 /* 0x183 = G, 1GB, R/W, P */ _exception_stack: .fill CONFIG_EXCEPTION_STACK_SIZE, 1, 0xAA +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 1) +.global _exception_stack1 +.align 16 +_exception_stack1: + .fill CONFIG_EXCEPTION_STACK_SIZE, 1, 0xAA +#endif + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 2) +.global _exception_stack2 +.align 16 +_exception_stack2: + .fill CONFIG_EXCEPTION_STACK_SIZE, 1, 0xAA +#endif + +#if defined(CONFIG_SMP) && (CONFIG_MP_NUM_CPUS > 3) +.global _exception_stack3 +.align 16 +_exception_stack3: + .fill CONFIG_EXCEPTION_STACK_SIZE, 1, 0xAA +#endif diff --git a/arch/x86/include/intel64/kernel_arch_data.h b/arch/x86/include/intel64/kernel_arch_data.h index 858672e3bfd..42979c6a8d9 100644 --- a/arch/x86/include/intel64/kernel_arch_data.h +++ b/arch/x86/include/intel64/kernel_arch_data.h @@ -25,6 +25,12 @@ #define X86_KERNEL_CPU0_GS 0x30 /* data selector covering TSS */ #define X86_KERNEL_CPU0_TR 0x40 /* 64-bit task state segment */ +#define X86_KERNEL_CPU1_GS 0x50 /* data selector covering TSS */ +#define X86_KERNEL_CPU1_TR 0x60 /* 64-bit task state segment */ +#define X86_KERNEL_CPU2_GS 0x70 /* data selector covering TSS */ +#define X86_KERNEL_CPU2_TR 0x80 /* 64-bit task state segment */ +#define X86_KERNEL_CPU3_GS 0x90 /* data selector covering TSS */ +#define X86_KERNEL_CPU3_TR 0xA0 /* 64-bit task state segment */ #ifndef _ASMLANGUAGE @@ -86,6 +92,8 @@ struct x86_cpuboot { typedef struct x86_cpuboot x86_cpuboot_t; +extern u8_t x86_cpu_loapics[]; /* CPU logical ID -> local APIC ID */ + #endif /* _ASMLANGUAGE */ #endif /* ZEPHYR_ARCH_X86_INCLUDE_INTEL64_KERNEL_ARCH_DATA_H_ */ diff --git a/arch/x86/include/kernel_arch_func.h b/arch/x86/include/kernel_arch_func.h index 33a7cf6d69c..6299aa12be8 100644 --- a/arch/x86/include/kernel_arch_func.h +++ b/arch/x86/include/kernel_arch_func.h @@ -21,6 +21,9 @@ #ifndef _ASMLANGUAGE extern K_THREAD_STACK_DEFINE(_interrupt_stack, CONFIG_ISR_STACK_SIZE); +extern K_THREAD_STACK_DEFINE(_interrupt_stack1, CONFIG_ISR_STACK_SIZE); +extern K_THREAD_STACK_DEFINE(_interrupt_stack2, CONFIG_ISR_STACK_SIZE); +extern K_THREAD_STACK_DEFINE(_interrupt_stack3, CONFIG_ISR_STACK_SIZE); #endif diff --git a/boards/x86/qemu_x86/qemu_x86_long_defconfig b/boards/x86/qemu_x86/qemu_x86_long_defconfig index 50fcdd2f5cb..7a8091ea7cb 100644 --- a/boards/x86/qemu_x86/qemu_x86_long_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_long_defconfig @@ -15,3 +15,4 @@ CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 CONFIG_TEST_RANDOM_GENERATOR=y CONFIG_DEBUG_INFO=y CONFIG_SMP=y +CONFIG_MP_NUM_CPUS=2 diff --git a/soc/x86/apollo_lake/CMakeLists.txt b/soc/x86/apollo_lake/CMakeLists.txt index 3b9e61016ef..b5f24f2ecd3 100644 --- a/soc/x86/apollo_lake/CMakeLists.txt +++ b/soc/x86/apollo_lake/CMakeLists.txt @@ -6,3 +6,4 @@ zephyr_library_include_directories(${ZEPHYR_BASE}/drivers) zephyr_cc_option(-march=goldmont) zephyr_library_sources(soc.c) +zephyr_library_sources_ifdef(CONFIG_SMP cpu.c) diff --git a/soc/x86/apollo_lake/cpu.c b/soc/x86/apollo_lake/cpu.c new file mode 100644 index 00000000000..388b52a5343 --- /dev/null +++ b/soc/x86/apollo_lake/cpu.c @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2019 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +u8_t x86_cpu_loapics[] = { 0x00, 0x02, 0x04, 0x06 };