diff --git a/arch/x86/core/CMakeLists.txt b/arch/x86/core/CMakeLists.txt index 06873a05212..a9c0202415e 100644 --- a/arch/x86/core/CMakeLists.txt +++ b/arch/x86/core/CMakeLists.txt @@ -17,6 +17,7 @@ zephyr_library_sources_if_kconfig(reboot_rst_cnt.c) zephyr_library_sources_if_kconfig(multiboot.c) zephyr_library_sources_if_kconfig(acpi.c) zephyr_library_sources_if_kconfig(x86_mmu.c) +zephyr_library_sources_if_kconfig(userspace.c) zephyr_library_sources_ifdef(CONFIG_X86_VERY_EARLY_CONSOLE early_serial.c) diff --git a/arch/x86/core/ia32/thread.c b/arch/x86/core/ia32/thread.c index 888affe055d..49e4a96aa6a 100644 --- a/arch/x86/core/ia32/thread.c +++ b/arch/x86/core/ia32/thread.c @@ -36,111 +36,6 @@ struct _x86_initial_frame { }; #ifdef CONFIG_X86_USERSPACE -/* Nothing to do here if KPTI is enabled. We are in supervisor mode, so the - * active PDPT is the kernel's page tables. If the incoming thread is in user - * mode we are going to switch CR3 to the thread- specific tables when we go - * through z_x86_trampoline_to_user. - * - * We don't need to update _main_tss either, privilege elevation always lands - * on the trampoline stack and the irq/sycall code has to manually transition - * off of it to the thread's kernel stack after switching page tables. - */ -#ifndef CONFIG_X86_KPTI -/* Change to new set of page tables. ONLY intended for use from - * z_x88_swap_update_page_tables(). This changes CR3, no memory access - * afterwards is legal unless it is known for sure that the relevant - * mappings are identical wrt supervisor mode until we iret out. - */ -static inline void page_tables_set(struct x86_page_tables *ptables) -{ - __asm__ volatile("movl %0, %%cr3\n\t" : : "r" (ptables) : "memory"); -} - -/* Update the to the incoming thread's page table, and update the location - * of the privilege elevation stack. - * - * May be called ONLY during context switch and when supervisor - * threads drop synchronously to user mode. Hot code path! - */ -void z_x86_swap_update_page_tables(struct k_thread *incoming) -{ - struct x86_page_tables *ptables = - z_x86_thread_page_tables_get(incoming); - - /* If we're a user thread, we want the active page tables to - * be the per-thread instance. - * - * However, if we're a supervisor thread, use the master - * kernel page tables instead. - */ - if ((incoming->base.user_options & K_USER) != 0) { - /* In case of privilege elevation, use the incoming - * thread's kernel stack. This area starts immediately - * before the PDPT. - */ - _main_tss.esp0 = (uintptr_t)(incoming->arch.psp); - } - - /* Check first that we actually need to do this, since setting - * CR3 involves an expensive full TLB flush. - */ - if (ptables != z_x86_page_tables_get()) { - page_tables_set(ptables); - } -} -#endif /* CONFIG_X86_KPTI */ - -static FUNC_NORETURN void drop_to_user(k_thread_entry_t user_entry, - void *p1, void *p2, void *p3) -{ - u32_t stack_end; - struct z_x86_thread_stack_header *header = - (struct z_x86_thread_stack_header *)_current->stack_obj; - - _current->arch.psp = - header->privilege_stack + sizeof(header->privilege_stack); - - /* Transition will reset stack pointer to initial, discarding - * any old context since this is a one-way operation - */ - stack_end = STACK_ROUND_DOWN(_current->stack_info.start + - _current->stack_info.size); - - z_x86_userspace_enter(user_entry, p1, p2, p3, stack_end, - _current->stack_info.start); - CODE_UNREACHABLE; -} - -FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, - void *p1, void *p2, void *p3) -{ - /* Initialize per-thread page tables, since that wasn't done when - * the thread was initialized (K_USER was not set at creation time) - */ - z_x86_thread_pt_init(_current); - - /* Apply memory domain configuration, if assigned */ - if (_current->mem_domain_info.mem_domain != NULL) { - z_x86_apply_mem_domain(z_x86_thread_page_tables_get(_current), - _current->mem_domain_info.mem_domain); - } - -#ifndef CONFIG_X86_KPTI - /* We're synchronously dropping into user mode from a thread that - * used to be in supervisor mode. K_USER flag has now been set, but - * Need to swap from the kernel's page tables to the per-thread page - * tables. - * - * Safe to update page tables from here, all tables are identity- - * mapped and memory areas used before the ring 3 transition all - * have the same attributes wrt supervisor mode access. - */ - z_x86_swap_update_page_tables(_current); -#endif - - drop_to_user(user_entry, p1, p2, p3); -} - /* Implemented in userspace.S */ extern void z_x86_syscall_entry_stub(void); @@ -148,7 +43,6 @@ extern void z_x86_syscall_entry_stub(void); * userspace can invoke it. */ NANO_CPU_INT_REGISTER(z_x86_syscall_entry_stub, -1, -1, 0x80, 3); - #endif /* CONFIG_X86_USERSPACE */ #if defined(CONFIG_FLOAT) && defined(CONFIG_FP_SHARING) @@ -172,6 +66,7 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, { char *stack_buf; char *stack_high; + void *swap_entry; struct _x86_initial_frame *initial_frame; Z_ASSERT_VALID_PRIO(priority, entry); @@ -188,6 +83,12 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, true); #endif +#ifdef CONFIG_USERSPACE + swap_entry = z_x86_userspace_prepare_thread(thread); +#else + swap_entry = z_thread_entry; +#endif + stack_high = (char *)STACK_ROUND_DOWN(stack_buf + stack_size); /* Create an initial context on the stack expected by z_swap() */ @@ -199,36 +100,19 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, initial_frame->p2 = parameter2; initial_frame->p3 = parameter3; initial_frame->eflags = EFLAGS_INITIAL; -#ifdef CONFIG_X86_USERSPACE - if ((options & K_USER) != 0U) { - z_x86_thread_pt_init(thread); #ifdef _THREAD_WRAPPER_REQUIRED - initial_frame->edi = (u32_t)drop_to_user; - initial_frame->thread_entry = z_x86_thread_entry_wrapper; + initial_frame->edi = (u32_t)swap_entry; + initial_frame->thread_entry = z_x86_thread_entry_wrapper; #else - initial_frame->thread_entry = drop_to_user; + initial_frame->thread_entry = swap_entry; #endif /* _THREAD_WRAPPER_REQUIRED */ - } else -#endif /* CONFIG_X86_USERSPACE */ - { -#ifdef CONFIG_X86_USERSPACE - thread->arch.ptables = &z_x86_kernel_ptables; -#endif -#ifdef _THREAD_WRAPPER_REQUIRED - initial_frame->edi = (u32_t)z_thread_entry; - initial_frame->thread_entry = z_x86_thread_entry_wrapper; -#else - initial_frame->thread_entry = z_thread_entry; -#endif - } + /* Remaining _x86_initial_frame members can be garbage, z_thread_entry() * doesn't care about their state when execution begins */ thread->callee_saved.esp = (unsigned long)initial_frame; - #if defined(CONFIG_LAZY_FP_SHARING) thread->arch.excNestCount = 0; #endif /* CONFIG_LAZY_FP_SHARING */ - thread->arch.flags = 0; } diff --git a/arch/x86/core/userspace.c b/arch/x86/core/userspace.c new file mode 100644 index 00000000000..2ef945c1c26 --- /dev/null +++ b/arch/x86/core/userspace.c @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2017 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +#ifndef CONFIG_X86_KPTI +/* Change to new set of page tables. ONLY intended for use from + * z_x88_swap_update_page_tables(). This changes CR3, no memory access + * afterwards is legal unless it is known for sure that the relevant + * mappings are identical wrt supervisor mode until we iret out. + */ +static inline void page_tables_set(struct x86_page_tables *ptables) +{ +#ifdef CONFIG_X86_64 + __asm__ volatile("movq %0, %%cr3\n\t" : : "r" (ptables) : "memory"); +#else + __asm__ volatile("movl %0, %%cr3\n\t" : : "r" (ptables) : "memory"); +#endif +} + +/* Set initial stack pointer for privilege mode elevations */ +static inline void set_initial_psp(char *psp) +{ +#ifdef CONFIG_X86_64 + __asm__ volatile("movq %0, %%gs:__x86_tss64_t_psp_OFFSET\n\t" + : : "r" (psp)); +#else + _main_tss.esp0 = (uintptr_t)psp; +#endif +} + +/* Update the to the incoming thread's page table, and update the location of + * the privilege elevation stack. + * + * May be called ONLY during context switch and when supervisor threads drop + * synchronously to user mode. Hot code path! + * + * Nothing to do here if KPTI is enabled. We are in supervisor mode, so the + * active page tables are the kernel's page tables. If the incoming thread is + * in user mode we are going to switch CR3 to the thread-specific tables when + * we go through z_x86_trampoline_to_user. + * + * We don't need to update the privilege mode initial stack pointer either, + * privilege elevation always lands on the trampoline stack and the irq/sycall + * code has to manually transition off of it to the thread's kernel stack after + * switching page tables. + */ +void z_x86_swap_update_page_tables(struct k_thread *incoming) +{ + struct x86_page_tables *ptables; + + if ((incoming->base.user_options & K_USER) != 0) { + set_initial_psp(incoming->arch.psp); + } + + /* Check first that we actually need to do this, since setting + * CR3 involves an expensive full TLB flush. + */ + ptables = z_x86_thread_page_tables_get(incoming); + + if (ptables != z_x86_page_tables_get()) { + page_tables_set(ptables); + } +} +#endif /* CONFIG_X86_KPTI */ + +FUNC_NORETURN static void drop_to_user(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3) +{ + u32_t stack_end; + + /* Transition will reset stack pointer to initial, discarding + * any old context since this is a one-way operation + */ + stack_end = STACK_ROUND_DOWN(_current->stack_info.start + + _current->stack_info.size); + + z_x86_userspace_enter(user_entry, p1, p2, p3, stack_end, + _current->stack_info.start); + CODE_UNREACHABLE; +} + +static inline void +set_privilege_stack_perms(struct z_x86_thread_stack_header *header, + bool is_usermode) +{ + /* Set MMU properties for the privilege mode elevation stack. If we're + * not in user mode, this functions as a guard area. + */ + z_x86_mmu_set_flags(&z_x86_kernel_ptables, &header->privilege_stack, + MMU_PAGE_SIZE, + is_usermode ? MMU_ENTRY_WRITE : MMU_ENTRY_READ, + Z_X86_MMU_RW, true); +} + +/* Does the following: + * + * - Allows the kernel to write to the privilege elevation stack area. + * - Initialize per-thread page tables and update thread->arch.ptables to + * point to them. + * - Set thread->arch.psp to point to the initial stack pointer for user + * mode privilege elevation for system calls; supervisor mode threads leave + * this uninitailized. + */ +static void prepare_user_thread(struct k_thread *thread) +{ + struct z_x86_thread_stack_header *header = + (struct z_x86_thread_stack_header *)thread->stack_obj; + + __ASSERT((thread->base.user_options & K_USER) != 0, + "not a user thread"); + + /* Set privileve elevation stack area to writable. Need to do this + * before calling z_x86_pt_init(), as on 32-bit the top-level PDPT + * is in there as well. + */ + set_privilege_stack_perms(header, true); + + /* Create and program into the MMU the per-thread page tables */ + z_x86_thread_pt_init(thread); + + thread->arch.psp = + header->privilege_stack + sizeof(header->privilege_stack); +} + +static void prepare_supervisor_thread(struct k_thread *thread) +{ + struct z_x86_thread_stack_header *header = + (struct z_x86_thread_stack_header *)thread->stack_obj; + + thread->arch.ptables = &z_x86_kernel_ptables; + + /* Privilege elevation stack set to read-only to function + * as a guard area. This gets made writable if we drop + * to user mode later. + */ + set_privilege_stack_perms(header, false); +} + +/* Preparation steps needed for all threads if user mode is turned on. + * + * Returns the initial entry point to swap into. + */ +void *z_x86_userspace_prepare_thread(struct k_thread *thread) +{ + void *initial_entry; + + if ((thread->base.user_options & K_USER) != 0U) { + prepare_user_thread(thread); + initial_entry = drop_to_user; + } else { + prepare_supervisor_thread(thread); + initial_entry = z_thread_entry; + } + + return initial_entry; +} + +FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3) +{ + prepare_user_thread(_current); + + /* Apply memory domain configuration, if assigned. Threads that + * started in user mode already had this done via z_setup_new_thread() + */ + if (_current->mem_domain_info.mem_domain != NULL) { + z_x86_apply_mem_domain(_current->arch.ptables, + _current->mem_domain_info.mem_domain); + } + +#ifndef CONFIG_X86_KPTI + /* We're synchronously dropping into user mode from a thread that + * used to be in supervisor mode. K_USER flag has now been set, but + * Need to swap from the kernel's page tables to the per-thread page + * tables. + * + * Safe to update page tables from here, all tables are identity- + * mapped and memory areas used before the ring 3 transition all + * have the same attributes wrt supervisor mode access. + * + * Threads that started in user mode already had this applied on + * initial context switch. + */ + z_x86_swap_update_page_tables(_current); +#endif + + drop_to_user(user_entry, p1, p2, p3); +} diff --git a/arch/x86/include/ia32/kernel_arch_func.h b/arch/x86/include/ia32/kernel_arch_func.h index 71b25e78e81..a0521fca3da 100644 --- a/arch/x86/include/ia32/kernel_arch_func.h +++ b/arch/x86/include/ia32/kernel_arch_func.h @@ -33,19 +33,6 @@ arch_thread_return_value_set(struct k_thread *thread, unsigned int value) extern void arch_cpu_atomic_idle(unsigned int key); -#ifdef CONFIG_USERSPACE -extern FUNC_NORETURN void z_x86_userspace_enter(k_thread_entry_t user_entry, - void *p1, void *p2, void *p3, - u32_t stack_end, - u32_t stack_start); - -void z_x86_thread_pt_init(struct k_thread *thread); - -void z_x86_apply_mem_domain(struct x86_page_tables *ptables, - struct k_mem_domain *mem_domain); - -#endif /* CONFIG_USERSPACE */ - /* ASM code to fiddle with registers to enable the MMU with PAE paging */ void z_x86_enable_paging(void); diff --git a/arch/x86/include/kernel_arch_func.h b/arch/x86/include/kernel_arch_func.h index 0ecf0aec237..7a702d91a6d 100644 --- a/arch/x86/include/kernel_arch_func.h +++ b/arch/x86/include/kernel_arch_func.h @@ -88,6 +88,26 @@ void z_x86_page_fault_handler(z_arch_esf_t *esf); */ bool z_x86_check_stack_bounds(uintptr_t addr, size_t size, u16_t cs); #endif /* CONFIG_THREAD_STACK_INFO */ + +#ifdef CONFIG_USERSPACE +extern FUNC_NORETURN void z_x86_userspace_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3, + uintptr_t stack_end, + uintptr_t stack_start); + +/* Preparation steps needed for all threads if user mode is turned on. + * + * Returns the initial entry point to swap into. + */ +void *z_x86_userspace_prepare_thread(struct k_thread *thread); + +void z_x86_thread_pt_init(struct k_thread *thread); + +void z_x86_apply_mem_domain(struct x86_page_tables *ptables, + struct k_mem_domain *mem_domain); + +#endif /* CONFIG_USERSPACE */ + #endif /* !_ASMLANGUAGE */ #endif /* ZEPHYR_ARCH_X86_INCLUDE_KERNEL_ARCH_FUNC_H_ */