x86: up-level some user mode functions
These are now common code, all are related to user mode threads. The rat's nest of ifdefs in ia32's arch_new_thread has been greatly simplified, there is now just one hook if user mode is turned on. Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
This commit is contained in:
Andrew Boie
Anas Nashif
parent
7ea958e0dd
commit
7f82b99ad4
5 changed files with 228 additions and 140 deletions
|
|
@ -17,6 +17,7 @@ zephyr_library_sources_if_kconfig(reboot_rst_cnt.c)
|
|||
zephyr_library_sources_if_kconfig(multiboot.c)
|
||||
zephyr_library_sources_if_kconfig(acpi.c)
|
||||
zephyr_library_sources_if_kconfig(x86_mmu.c)
|
||||
zephyr_library_sources_if_kconfig(userspace.c)
|
||||
|
||||
zephyr_library_sources_ifdef(CONFIG_X86_VERY_EARLY_CONSOLE early_serial.c)
|
||||
|
||||
|
|
|
|||
|
|
@ -36,111 +36,6 @@ struct _x86_initial_frame {
|
|||
};
|
||||
|
||||
#ifdef CONFIG_X86_USERSPACE
|
||||
/* Nothing to do here if KPTI is enabled. We are in supervisor mode, so the
|
||||
* active PDPT is the kernel's page tables. If the incoming thread is in user
|
||||
* mode we are going to switch CR3 to the thread- specific tables when we go
|
||||
* through z_x86_trampoline_to_user.
|
||||
*
|
||||
* We don't need to update _main_tss either, privilege elevation always lands
|
||||
* on the trampoline stack and the irq/sycall code has to manually transition
|
||||
* off of it to the thread's kernel stack after switching page tables.
|
||||
*/
|
||||
#ifndef CONFIG_X86_KPTI
|
||||
/* Change to new set of page tables. ONLY intended for use from
|
||||
* z_x88_swap_update_page_tables(). This changes CR3, no memory access
|
||||
* afterwards is legal unless it is known for sure that the relevant
|
||||
* mappings are identical wrt supervisor mode until we iret out.
|
||||
*/
|
||||
static inline void page_tables_set(struct x86_page_tables *ptables)
|
||||
{
|
||||
__asm__ volatile("movl %0, %%cr3\n\t" : : "r" (ptables) : "memory");
|
||||
}
|
||||
|
||||
/* Update the to the incoming thread's page table, and update the location
|
||||
* of the privilege elevation stack.
|
||||
*
|
||||
* May be called ONLY during context switch and when supervisor
|
||||
* threads drop synchronously to user mode. Hot code path!
|
||||
*/
|
||||
void z_x86_swap_update_page_tables(struct k_thread *incoming)
|
||||
{
|
||||
struct x86_page_tables *ptables =
|
||||
z_x86_thread_page_tables_get(incoming);
|
||||
|
||||
/* If we're a user thread, we want the active page tables to
|
||||
* be the per-thread instance.
|
||||
*
|
||||
* However, if we're a supervisor thread, use the master
|
||||
* kernel page tables instead.
|
||||
*/
|
||||
if ((incoming->base.user_options & K_USER) != 0) {
|
||||
/* In case of privilege elevation, use the incoming
|
||||
* thread's kernel stack. This area starts immediately
|
||||
* before the PDPT.
|
||||
*/
|
||||
_main_tss.esp0 = (uintptr_t)(incoming->arch.psp);
|
||||
}
|
||||
|
||||
/* Check first that we actually need to do this, since setting
|
||||
* CR3 involves an expensive full TLB flush.
|
||||
*/
|
||||
if (ptables != z_x86_page_tables_get()) {
|
||||
page_tables_set(ptables);
|
||||
}
|
||||
}
|
||||
#endif /* CONFIG_X86_KPTI */
|
||||
|
||||
static FUNC_NORETURN void drop_to_user(k_thread_entry_t user_entry,
|
||||
void *p1, void *p2, void *p3)
|
||||
{
|
||||
u32_t stack_end;
|
||||
struct z_x86_thread_stack_header *header =
|
||||
(struct z_x86_thread_stack_header *)_current->stack_obj;
|
||||
|
||||
_current->arch.psp =
|
||||
header->privilege_stack + sizeof(header->privilege_stack);
|
||||
|
||||
/* Transition will reset stack pointer to initial, discarding
|
||||
* any old context since this is a one-way operation
|
||||
*/
|
||||
stack_end = STACK_ROUND_DOWN(_current->stack_info.start +
|
||||
_current->stack_info.size);
|
||||
|
||||
z_x86_userspace_enter(user_entry, p1, p2, p3, stack_end,
|
||||
_current->stack_info.start);
|
||||
CODE_UNREACHABLE;
|
||||
}
|
||||
|
||||
FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
||||
void *p1, void *p2, void *p3)
|
||||
{
|
||||
/* Initialize per-thread page tables, since that wasn't done when
|
||||
* the thread was initialized (K_USER was not set at creation time)
|
||||
*/
|
||||
z_x86_thread_pt_init(_current);
|
||||
|
||||
/* Apply memory domain configuration, if assigned */
|
||||
if (_current->mem_domain_info.mem_domain != NULL) {
|
||||
z_x86_apply_mem_domain(z_x86_thread_page_tables_get(_current),
|
||||
_current->mem_domain_info.mem_domain);
|
||||
}
|
||||
|
||||
#ifndef CONFIG_X86_KPTI
|
||||
/* We're synchronously dropping into user mode from a thread that
|
||||
* used to be in supervisor mode. K_USER flag has now been set, but
|
||||
* Need to swap from the kernel's page tables to the per-thread page
|
||||
* tables.
|
||||
*
|
||||
* Safe to update page tables from here, all tables are identity-
|
||||
* mapped and memory areas used before the ring 3 transition all
|
||||
* have the same attributes wrt supervisor mode access.
|
||||
*/
|
||||
z_x86_swap_update_page_tables(_current);
|
||||
#endif
|
||||
|
||||
drop_to_user(user_entry, p1, p2, p3);
|
||||
}
|
||||
|
||||
/* Implemented in userspace.S */
|
||||
extern void z_x86_syscall_entry_stub(void);
|
||||
|
||||
|
|
@ -148,7 +43,6 @@ extern void z_x86_syscall_entry_stub(void);
|
|||
* userspace can invoke it.
|
||||
*/
|
||||
NANO_CPU_INT_REGISTER(z_x86_syscall_entry_stub, -1, -1, 0x80, 3);
|
||||
|
||||
#endif /* CONFIG_X86_USERSPACE */
|
||||
|
||||
#if defined(CONFIG_FLOAT) && defined(CONFIG_FP_SHARING)
|
||||
|
|
@ -172,6 +66,7 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
|
|||
{
|
||||
char *stack_buf;
|
||||
char *stack_high;
|
||||
void *swap_entry;
|
||||
struct _x86_initial_frame *initial_frame;
|
||||
|
||||
Z_ASSERT_VALID_PRIO(priority, entry);
|
||||
|
|
@ -188,6 +83,12 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
|
|||
true);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
swap_entry = z_x86_userspace_prepare_thread(thread);
|
||||
#else
|
||||
swap_entry = z_thread_entry;
|
||||
#endif
|
||||
|
||||
stack_high = (char *)STACK_ROUND_DOWN(stack_buf + stack_size);
|
||||
|
||||
/* Create an initial context on the stack expected by z_swap() */
|
||||
|
|
@ -199,36 +100,19 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
|
|||
initial_frame->p2 = parameter2;
|
||||
initial_frame->p3 = parameter3;
|
||||
initial_frame->eflags = EFLAGS_INITIAL;
|
||||
#ifdef CONFIG_X86_USERSPACE
|
||||
if ((options & K_USER) != 0U) {
|
||||
z_x86_thread_pt_init(thread);
|
||||
#ifdef _THREAD_WRAPPER_REQUIRED
|
||||
initial_frame->edi = (u32_t)drop_to_user;
|
||||
initial_frame->thread_entry = z_x86_thread_entry_wrapper;
|
||||
initial_frame->edi = (u32_t)swap_entry;
|
||||
initial_frame->thread_entry = z_x86_thread_entry_wrapper;
|
||||
#else
|
||||
initial_frame->thread_entry = drop_to_user;
|
||||
initial_frame->thread_entry = swap_entry;
|
||||
#endif /* _THREAD_WRAPPER_REQUIRED */
|
||||
} else
|
||||
#endif /* CONFIG_X86_USERSPACE */
|
||||
{
|
||||
#ifdef CONFIG_X86_USERSPACE
|
||||
thread->arch.ptables = &z_x86_kernel_ptables;
|
||||
#endif
|
||||
#ifdef _THREAD_WRAPPER_REQUIRED
|
||||
initial_frame->edi = (u32_t)z_thread_entry;
|
||||
initial_frame->thread_entry = z_x86_thread_entry_wrapper;
|
||||
#else
|
||||
initial_frame->thread_entry = z_thread_entry;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Remaining _x86_initial_frame members can be garbage, z_thread_entry()
|
||||
* doesn't care about their state when execution begins
|
||||
*/
|
||||
thread->callee_saved.esp = (unsigned long)initial_frame;
|
||||
|
||||
#if defined(CONFIG_LAZY_FP_SHARING)
|
||||
thread->arch.excNestCount = 0;
|
||||
#endif /* CONFIG_LAZY_FP_SHARING */
|
||||
|
||||
thread->arch.flags = 0;
|
||||
}
|
||||
|
|
|
|||
196
arch/x86/core/userspace.c
Normal file
196
arch/x86/core/userspace.c
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
/*
|
||||
* Copyright (c) 2017 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <kernel.h>
|
||||
#include <sys/speculation.h>
|
||||
#include <syscall_handler.h>
|
||||
#include <kernel_arch_func.h>
|
||||
#include <ksched.h>
|
||||
|
||||
#ifndef CONFIG_X86_KPTI
|
||||
/* Change to new set of page tables. ONLY intended for use from
|
||||
* z_x88_swap_update_page_tables(). This changes CR3, no memory access
|
||||
* afterwards is legal unless it is known for sure that the relevant
|
||||
* mappings are identical wrt supervisor mode until we iret out.
|
||||
*/
|
||||
static inline void page_tables_set(struct x86_page_tables *ptables)
|
||||
{
|
||||
#ifdef CONFIG_X86_64
|
||||
__asm__ volatile("movq %0, %%cr3\n\t" : : "r" (ptables) : "memory");
|
||||
#else
|
||||
__asm__ volatile("movl %0, %%cr3\n\t" : : "r" (ptables) : "memory");
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Set initial stack pointer for privilege mode elevations */
|
||||
static inline void set_initial_psp(char *psp)
|
||||
{
|
||||
#ifdef CONFIG_X86_64
|
||||
__asm__ volatile("movq %0, %%gs:__x86_tss64_t_psp_OFFSET\n\t"
|
||||
: : "r" (psp));
|
||||
#else
|
||||
_main_tss.esp0 = (uintptr_t)psp;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Update the to the incoming thread's page table, and update the location of
|
||||
* the privilege elevation stack.
|
||||
*
|
||||
* May be called ONLY during context switch and when supervisor threads drop
|
||||
* synchronously to user mode. Hot code path!
|
||||
*
|
||||
* Nothing to do here if KPTI is enabled. We are in supervisor mode, so the
|
||||
* active page tables are the kernel's page tables. If the incoming thread is
|
||||
* in user mode we are going to switch CR3 to the thread-specific tables when
|
||||
* we go through z_x86_trampoline_to_user.
|
||||
*
|
||||
* We don't need to update the privilege mode initial stack pointer either,
|
||||
* privilege elevation always lands on the trampoline stack and the irq/sycall
|
||||
* code has to manually transition off of it to the thread's kernel stack after
|
||||
* switching page tables.
|
||||
*/
|
||||
void z_x86_swap_update_page_tables(struct k_thread *incoming)
|
||||
{
|
||||
struct x86_page_tables *ptables;
|
||||
|
||||
if ((incoming->base.user_options & K_USER) != 0) {
|
||||
set_initial_psp(incoming->arch.psp);
|
||||
}
|
||||
|
||||
/* Check first that we actually need to do this, since setting
|
||||
* CR3 involves an expensive full TLB flush.
|
||||
*/
|
||||
ptables = z_x86_thread_page_tables_get(incoming);
|
||||
|
||||
if (ptables != z_x86_page_tables_get()) {
|
||||
page_tables_set(ptables);
|
||||
}
|
||||
}
|
||||
#endif /* CONFIG_X86_KPTI */
|
||||
|
||||
FUNC_NORETURN static void drop_to_user(k_thread_entry_t user_entry,
|
||||
void *p1, void *p2, void *p3)
|
||||
{
|
||||
u32_t stack_end;
|
||||
|
||||
/* Transition will reset stack pointer to initial, discarding
|
||||
* any old context since this is a one-way operation
|
||||
*/
|
||||
stack_end = STACK_ROUND_DOWN(_current->stack_info.start +
|
||||
_current->stack_info.size);
|
||||
|
||||
z_x86_userspace_enter(user_entry, p1, p2, p3, stack_end,
|
||||
_current->stack_info.start);
|
||||
CODE_UNREACHABLE;
|
||||
}
|
||||
|
||||
static inline void
|
||||
set_privilege_stack_perms(struct z_x86_thread_stack_header *header,
|
||||
bool is_usermode)
|
||||
{
|
||||
/* Set MMU properties for the privilege mode elevation stack. If we're
|
||||
* not in user mode, this functions as a guard area.
|
||||
*/
|
||||
z_x86_mmu_set_flags(&z_x86_kernel_ptables, &header->privilege_stack,
|
||||
MMU_PAGE_SIZE,
|
||||
is_usermode ? MMU_ENTRY_WRITE : MMU_ENTRY_READ,
|
||||
Z_X86_MMU_RW, true);
|
||||
}
|
||||
|
||||
/* Does the following:
|
||||
*
|
||||
* - Allows the kernel to write to the privilege elevation stack area.
|
||||
* - Initialize per-thread page tables and update thread->arch.ptables to
|
||||
* point to them.
|
||||
* - Set thread->arch.psp to point to the initial stack pointer for user
|
||||
* mode privilege elevation for system calls; supervisor mode threads leave
|
||||
* this uninitailized.
|
||||
*/
|
||||
static void prepare_user_thread(struct k_thread *thread)
|
||||
{
|
||||
struct z_x86_thread_stack_header *header =
|
||||
(struct z_x86_thread_stack_header *)thread->stack_obj;
|
||||
|
||||
__ASSERT((thread->base.user_options & K_USER) != 0,
|
||||
"not a user thread");
|
||||
|
||||
/* Set privileve elevation stack area to writable. Need to do this
|
||||
* before calling z_x86_pt_init(), as on 32-bit the top-level PDPT
|
||||
* is in there as well.
|
||||
*/
|
||||
set_privilege_stack_perms(header, true);
|
||||
|
||||
/* Create and program into the MMU the per-thread page tables */
|
||||
z_x86_thread_pt_init(thread);
|
||||
|
||||
thread->arch.psp =
|
||||
header->privilege_stack + sizeof(header->privilege_stack);
|
||||
}
|
||||
|
||||
static void prepare_supervisor_thread(struct k_thread *thread)
|
||||
{
|
||||
struct z_x86_thread_stack_header *header =
|
||||
(struct z_x86_thread_stack_header *)thread->stack_obj;
|
||||
|
||||
thread->arch.ptables = &z_x86_kernel_ptables;
|
||||
|
||||
/* Privilege elevation stack set to read-only to function
|
||||
* as a guard area. This gets made writable if we drop
|
||||
* to user mode later.
|
||||
*/
|
||||
set_privilege_stack_perms(header, false);
|
||||
}
|
||||
|
||||
/* Preparation steps needed for all threads if user mode is turned on.
|
||||
*
|
||||
* Returns the initial entry point to swap into.
|
||||
*/
|
||||
void *z_x86_userspace_prepare_thread(struct k_thread *thread)
|
||||
{
|
||||
void *initial_entry;
|
||||
|
||||
if ((thread->base.user_options & K_USER) != 0U) {
|
||||
prepare_user_thread(thread);
|
||||
initial_entry = drop_to_user;
|
||||
} else {
|
||||
prepare_supervisor_thread(thread);
|
||||
initial_entry = z_thread_entry;
|
||||
}
|
||||
|
||||
return initial_entry;
|
||||
}
|
||||
|
||||
FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
||||
void *p1, void *p2, void *p3)
|
||||
{
|
||||
prepare_user_thread(_current);
|
||||
|
||||
/* Apply memory domain configuration, if assigned. Threads that
|
||||
* started in user mode already had this done via z_setup_new_thread()
|
||||
*/
|
||||
if (_current->mem_domain_info.mem_domain != NULL) {
|
||||
z_x86_apply_mem_domain(_current->arch.ptables,
|
||||
_current->mem_domain_info.mem_domain);
|
||||
}
|
||||
|
||||
#ifndef CONFIG_X86_KPTI
|
||||
/* We're synchronously dropping into user mode from a thread that
|
||||
* used to be in supervisor mode. K_USER flag has now been set, but
|
||||
* Need to swap from the kernel's page tables to the per-thread page
|
||||
* tables.
|
||||
*
|
||||
* Safe to update page tables from here, all tables are identity-
|
||||
* mapped and memory areas used before the ring 3 transition all
|
||||
* have the same attributes wrt supervisor mode access.
|
||||
*
|
||||
* Threads that started in user mode already had this applied on
|
||||
* initial context switch.
|
||||
*/
|
||||
z_x86_swap_update_page_tables(_current);
|
||||
#endif
|
||||
|
||||
drop_to_user(user_entry, p1, p2, p3);
|
||||
}
|
||||
|
|
@ -33,19 +33,6 @@ arch_thread_return_value_set(struct k_thread *thread, unsigned int value)
|
|||
|
||||
extern void arch_cpu_atomic_idle(unsigned int key);
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
extern FUNC_NORETURN void z_x86_userspace_enter(k_thread_entry_t user_entry,
|
||||
void *p1, void *p2, void *p3,
|
||||
u32_t stack_end,
|
||||
u32_t stack_start);
|
||||
|
||||
void z_x86_thread_pt_init(struct k_thread *thread);
|
||||
|
||||
void z_x86_apply_mem_domain(struct x86_page_tables *ptables,
|
||||
struct k_mem_domain *mem_domain);
|
||||
|
||||
#endif /* CONFIG_USERSPACE */
|
||||
|
||||
/* ASM code to fiddle with registers to enable the MMU with PAE paging */
|
||||
void z_x86_enable_paging(void);
|
||||
|
||||
|
|
|
|||
|
|
@ -88,6 +88,26 @@ void z_x86_page_fault_handler(z_arch_esf_t *esf);
|
|||
*/
|
||||
bool z_x86_check_stack_bounds(uintptr_t addr, size_t size, u16_t cs);
|
||||
#endif /* CONFIG_THREAD_STACK_INFO */
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
extern FUNC_NORETURN void z_x86_userspace_enter(k_thread_entry_t user_entry,
|
||||
void *p1, void *p2, void *p3,
|
||||
uintptr_t stack_end,
|
||||
uintptr_t stack_start);
|
||||
|
||||
/* Preparation steps needed for all threads if user mode is turned on.
|
||||
*
|
||||
* Returns the initial entry point to swap into.
|
||||
*/
|
||||
void *z_x86_userspace_prepare_thread(struct k_thread *thread);
|
||||
|
||||
void z_x86_thread_pt_init(struct k_thread *thread);
|
||||
|
||||
void z_x86_apply_mem_domain(struct x86_page_tables *ptables,
|
||||
struct k_mem_domain *mem_domain);
|
||||
|
||||
#endif /* CONFIG_USERSPACE */
|
||||
|
||||
#endif /* !_ASMLANGUAGE */
|
||||
|
||||
#endif /* ZEPHYR_ARCH_X86_INCLUDE_KERNEL_ARCH_FUNC_H_ */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue