From 7ea958e0dd2133c7531ff0a14098ea3f843b3f76 Mon Sep 17 00:00:00 2001 From: Andrew Boie Date: Tue, 19 Nov 2019 15:08:49 -0800 Subject: [PATCH] x86: optimize locations of psp and thread ptables z_x86_thread_page_tables_get() now works for both user and supervisor threads, returning the kernel page tables in the latter case. This API has been up-leveled to a common header. The per-thread privilege elevation stack initial stack pointer, and the per-thread page table locations are no longer computed from other values, and instead are stored in thread->arch. A problem where the wrong page tables were dumped out on certain kinds of page faults has been fixed. Signed-off-by: Andrew Boie --- arch/x86/core/fatal.c | 10 +-------- arch/x86/core/ia32/thread.c | 17 +++++++++----- arch/x86/core/ia32/userspace.S | 28 +++++------------------- arch/x86/core/offsets/ia32_offsets.c | 2 ++ arch/x86/core/x86_mmu.c | 19 +++++++++++----- arch/x86/include/ia32/kernel_arch_func.h | 8 ------- arch/x86/include/kernel_arch_func.h | 11 ++++++++++ arch/x86/include/offsets_short_arch.h | 8 +++++++ include/arch/x86/ia32/thread.h | 13 +++++++++++ 9 files changed, 66 insertions(+), 50 deletions(-) diff --git a/arch/x86/core/fatal.c b/arch/x86/core/fatal.c index d7c7e2b20df..ae521d185f6 100644 --- a/arch/x86/core/fatal.c +++ b/arch/x86/core/fatal.c @@ -274,15 +274,7 @@ static void dump_page_fault(z_arch_esf_t *esf) } #ifdef CONFIG_X86_MMU -#ifdef CONFIG_USERSPACE - if (err & US) { - z_x86_dump_mmu_flags(z_x86_thread_page_tables_get(_current), - cr2); - } else -#endif /* CONFIG_USERSPACE */ - { - z_x86_dump_mmu_flags(&z_x86_kernel_ptables, cr2); - } + z_x86_dump_mmu_flags(z_x86_thread_page_tables_get(_current), cr2); #endif /* CONFIG_X86_MMU */ } #endif /* CONFIG_EXCEPTION_DEBUG */ diff --git a/arch/x86/core/ia32/thread.c b/arch/x86/core/ia32/thread.c index dd06f7f553b..888affe055d 100644 --- a/arch/x86/core/ia32/thread.c +++ b/arch/x86/core/ia32/thread.c @@ -64,7 +64,8 @@ static inline void page_tables_set(struct x86_page_tables *ptables) */ void z_x86_swap_update_page_tables(struct k_thread *incoming) { - struct x86_page_tables *ptables; + struct x86_page_tables *ptables = + z_x86_thread_page_tables_get(incoming); /* If we're a user thread, we want the active page tables to * be the per-thread instance. @@ -73,15 +74,11 @@ void z_x86_swap_update_page_tables(struct k_thread *incoming) * kernel page tables instead. */ if ((incoming->base.user_options & K_USER) != 0) { - ptables = z_x86_thread_page_tables_get(incoming); - /* In case of privilege elevation, use the incoming * thread's kernel stack. This area starts immediately * before the PDPT. */ - _main_tss.esp0 = (uintptr_t)ptables; - } else { - ptables = &z_x86_kernel_ptables; + _main_tss.esp0 = (uintptr_t)(incoming->arch.psp); } /* Check first that we actually need to do this, since setting @@ -97,6 +94,11 @@ static FUNC_NORETURN void drop_to_user(k_thread_entry_t user_entry, void *p1, void *p2, void *p3) { u32_t stack_end; + struct z_x86_thread_stack_header *header = + (struct z_x86_thread_stack_header *)_current->stack_obj; + + _current->arch.psp = + header->privilege_stack + sizeof(header->privilege_stack); /* Transition will reset stack pointer to initial, discarding * any old context since this is a one-way operation @@ -209,6 +211,9 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, } else #endif /* CONFIG_X86_USERSPACE */ { +#ifdef CONFIG_X86_USERSPACE + thread->arch.ptables = &z_x86_kernel_ptables; +#endif #ifdef _THREAD_WRAPPER_REQUIRED initial_frame->edi = (u32_t)z_thread_entry; initial_frame->thread_entry = z_x86_thread_entry_wrapper; diff --git a/arch/x86/core/ia32/userspace.S b/arch/x86/core/ia32/userspace.S index c37fe920888..399e66e59c3 100644 --- a/arch/x86/core/ia32/userspace.S +++ b/arch/x86/core/ia32/userspace.S @@ -55,16 +55,10 @@ SECTION_FUNC(TEXT, z_x86_trampoline_to_kernel) /* Save old trampoline stack pointer in %edi */ movl %esp, %edi - /* %esp = _kernel->current->stack_info.start - * - * This is the lowest address of the user mode stack, the PDPT is - * immediately before it, and then the highest address of the kernel - * stack. We want to transplant context here. - */ + /* Switch to privilege mode stack */ movl $_kernel, %esi movl _kernel_offset_to_current(%esi), %esi - movl _thread_offset_to_stack_start(%esi), %esp - subl $Z_X86_PDPT_SIZE, %esp + movl _thread_offset_to_psp(%esi), %esp /* Transplant stack context and restore ESI/EDI. Taking care to zero * or put uninteresting values where we stashed ESI/EDI since the @@ -134,15 +128,11 @@ SECTION_FUNC(TEXT, z_x86_trampoline_to_user_always) xchgl %edi, (%edi) /* Exchange old edi to restore it and put trampoline stack address in its old storage area */ - /* Switch to user page table. The per-thread user page table is - * located at the highest addresses of the privilege mode elevation - * stack, immediately below the thread stack buffer. - */ + /* Switch to user page table */ pushl %eax movl $_kernel, %eax movl _kernel_offset_to_current(%eax), %eax - movl _thread_offset_to_stack_start(%eax), %eax - subl $Z_X86_PDPT_SIZE, %eax + movl _thread_offset_to_ptables(%eax), %eax movl %eax, %cr3 popl %eax movl $0, -4(%esp) /* Delete stashed EAX data */ @@ -170,16 +160,10 @@ SECTION_FUNC(TEXT, z_x86_syscall_entry_stub) /* Save old trampoline stack pointer in %edi */ movl %esp, %edi - /* %esp = _kernel->current->stack_info.start - * - * This is the lowest address of the user mode stack, the PDPT is - * immediately before it, and then the highest address of the kernel - * stack. We want to transplant context here. - */ + /* Switch to privilege elevation stack */ movl $_kernel, %esi movl _kernel_offset_to_current(%esi), %esi - movl _thread_offset_to_stack_start(%esi), %esp - subl $Z_X86_PDPT_SIZE, %esp + movl _thread_offset_to_psp(%esi), %esp /* Transplant context according to layout above. Variant of logic * in x86_trampoline_to_kernel */ diff --git a/arch/x86/core/offsets/ia32_offsets.c b/arch/x86/core/offsets/ia32_offsets.c index 3ae3a09c691..0cbe9a8382c 100644 --- a/arch/x86/core/offsets/ia32_offsets.c +++ b/arch/x86/core/offsets/ia32_offsets.c @@ -31,6 +31,8 @@ GEN_OFFSET_SYM(_thread_arch_t, excNestCount); #endif #ifdef CONFIG_USERSPACE +GEN_OFFSET_SYM(_thread_arch_t, psp); +GEN_OFFSET_SYM(_thread_arch_t, ptables); GEN_ABSOLUTE_SYM(Z_X86_PDPT_SIZE, sizeof(struct x86_mmu_pdpt)); #endif diff --git a/arch/x86/core/x86_mmu.c b/arch/x86/core/x86_mmu.c index 2c3273f4032..27dfcdacb5a 100644 --- a/arch/x86/core/x86_mmu.c +++ b/arch/x86/core/x86_mmu.c @@ -990,11 +990,10 @@ static uintptr_t thread_pt_create(uintptr_t pages, * | ... | */ static void copy_page_tables(struct k_thread *thread, + struct x86_page_tables *thread_ptables, struct x86_page_tables *master_ptables) { uintptr_t pos, start; - struct x86_page_tables *thread_ptables = - z_x86_thread_page_tables_get(thread); struct z_x86_thread_stack_header *header = (struct z_x86_thread_stack_header *)thread->stack_obj; @@ -1091,11 +1090,21 @@ void z_x86_apply_mem_domain(struct x86_page_tables *ptables, * user mode. * * Sets up the per-thread page tables, such that when they are activated on - * context switch, everything is ready to go. + * context switch, everything is ready to go. thread->arch.ptables is updated + * to the thread-level tables instead of the kernel's page tbales. */ void z_x86_thread_pt_init(struct k_thread *thread) { - struct x86_page_tables *ptables = z_x86_thread_page_tables_get(thread); + struct x86_page_tables *ptables; + struct z_x86_thread_stack_header *header = + (struct z_x86_thread_stack_header *)thread->stack_obj; + +#ifdef CONFIG_X86_64 + ptables = (struct x86_page_tables *)(&header->page_tables); +#else + ptables = &header->kernel_data.ptables; +#endif + thread->arch.ptables = ptables; /* USER_PDPT contains the page tables with the boot time memory * policy. We use it as a template to set up the per-thread page @@ -1106,7 +1115,7 @@ void z_x86_thread_pt_init(struct k_thread *thread) * accessible pages except the trampoline page marked as non-present. * Without KPTI, they are the same object. */ - copy_page_tables(thread, &USER_PTABLES); + copy_page_tables(thread, ptables, &USER_PTABLES); /* Enable access to the thread's own stack buffer */ z_x86_mmu_set_flags(ptables, (void *)thread->stack_info.start, diff --git a/arch/x86/include/ia32/kernel_arch_func.h b/arch/x86/include/ia32/kernel_arch_func.h index 4410a88f94b..71b25e78e81 100644 --- a/arch/x86/include/ia32/kernel_arch_func.h +++ b/arch/x86/include/ia32/kernel_arch_func.h @@ -44,14 +44,6 @@ void z_x86_thread_pt_init(struct k_thread *thread); void z_x86_apply_mem_domain(struct x86_page_tables *ptables, struct k_mem_domain *mem_domain); -static inline struct x86_page_tables * -z_x86_thread_page_tables_get(struct k_thread *thread) -{ - struct z_x86_thread_stack_header *header = - (struct z_x86_thread_stack_header *)thread->stack_obj; - - return &header->kernel_data.ptables; -} #endif /* CONFIG_USERSPACE */ /* ASM code to fiddle with registers to enable the MMU with PAE paging */ diff --git a/arch/x86/include/kernel_arch_func.h b/arch/x86/include/kernel_arch_func.h index 857913b206f..0ecf0aec237 100644 --- a/arch/x86/include/kernel_arch_func.h +++ b/arch/x86/include/kernel_arch_func.h @@ -7,6 +7,7 @@ #define ZEPHYR_ARCH_X86_INCLUDE_KERNEL_ARCH_FUNC_H_ #include +#include #ifdef CONFIG_X86_64 #include @@ -45,6 +46,16 @@ void z_x86_early_serial_init(void); #ifdef CONFIG_X86_MMU /* Create all page tables with boot configuration and enable paging */ void z_x86_paging_init(void); + +static inline struct x86_page_tables * +z_x86_thread_page_tables_get(struct k_thread *thread) +{ +#ifdef CONFIG_USERSPACE + return thread->arch.ptables; +#else + return &z_x86_kernel_ptables; +#endif +} #endif /* CONFIG_X86_MMU */ /* Called upon CPU exception that is unhandled and hence fatal; dump diff --git a/arch/x86/include/offsets_short_arch.h b/arch/x86/include/offsets_short_arch.h index 8eec8ea4f20..a117939b7c1 100644 --- a/arch/x86/include/offsets_short_arch.h +++ b/arch/x86/include/offsets_short_arch.h @@ -15,4 +15,12 @@ #define _thread_offset_to_flags \ (___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET) +#ifdef CONFIG_USERSPACE +#define _thread_offset_to_psp \ + (___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET) + +#define _thread_offset_to_ptables \ + (___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET) +#endif /* CONFIG_USERSPACE */ + #endif /* ZEPHYR_ARCH_X86_INCLUDE_OFFSETS_SHORT_ARCH_H_ */ diff --git a/include/arch/x86/ia32/thread.h b/include/arch/x86/ia32/thread.h index c914bf4331e..816ab2be369 100644 --- a/include/arch/x86/ia32/thread.h +++ b/include/arch/x86/ia32/thread.h @@ -208,6 +208,19 @@ typedef struct s_preempFloatReg { struct _thread_arch { u8_t flags; +#ifdef CONFIG_USERSPACE + /* Pointer to page tables used by this thread. Supervisor threads + * always use the kernel's page table, user thread use per-thread + * tables stored in the stack object + */ + struct x86_page_tables *ptables; + + /* Initial privilege mode stack pointer when doing a system call. + * Un-set for supervisor threads. + */ + char *psp; +#endif + #if defined(CONFIG_LAZY_FP_SHARING) /* * Nested exception count to maintain setting of EXC_ACTIVE flag across