michaelh/zephyr
1
0
Fork 0
Code Releases Activity

x86: optimize locations of psp and thread ptables

Browse source 
z_x86_thread_page_tables_get() now works for both user
and supervisor threads, returning the kernel page tables
in the latter case. This API has been up-leveled to
a common header.

The per-thread privilege elevation stack initial stack
pointer, and the per-thread page table locations are no
longer computed from other values, and instead are stored
in thread->arch.

A problem where the wrong page tables were dumped out
on certain kinds of page faults has been fixed.

Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
This commit is contained in:
Andrew Boie 2019-11-19 15:08:49 -08:00 • committed by Anas Nashif
commit 7ea958e0dd
9 changed files with 66 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

10
arch/x86/core/fatal.c
View file

@ -274,15 +274,7 @@ static void dump_page_fault(z_arch_esf_t *esf)
}
#ifdef CONFIG_X86_MMU
#ifdef CONFIG_USERSPACE
if (err & US) {
z_x86_dump_mmu_flags(z_x86_thread_page_tables_get(_current),
cr2);
} else
#endif /* CONFIG_USERSPACE */
{
z_x86_dump_mmu_flags(&z_x86_kernel_ptables, cr2);
}
z_x86_dump_mmu_flags(z_x86_thread_page_tables_get(_current), cr2);
#endif /* CONFIG_X86_MMU */
}
#endif /* CONFIG_EXCEPTION_DEBUG */

17
arch/x86/core/ia32/thread.c
View file

@ -64,7 +64,8 @@ static inline void page_tables_set(struct x86_page_tables *ptables)
*/
void z_x86_swap_update_page_tables(struct k_thread *incoming)
{
struct x86_page_tables *ptables;
struct x86_page_tables *ptables =
z_x86_thread_page_tables_get(incoming);
/* If we're a user thread, we want the active page tables to
* be the per-thread instance.
@ -73,15 +74,11 @@ void z_x86_swap_update_page_tables(struct k_thread *incoming)
* kernel page tables instead.
*/
if ((incoming->base.user_options & K_USER) != 0) {
ptables = z_x86_thread_page_tables_get(incoming);
/* In case of privilege elevation, use the incoming
* thread's kernel stack. This area starts immediately
* before the PDPT.
*/
_main_tss.esp0 = (uintptr_t)ptables;
} else {
ptables = &z_x86_kernel_ptables;
_main_tss.esp0 = (uintptr_t)(incoming->arch.psp);
}
/* Check first that we actually need to do this, since setting
@ -97,6 +94,11 @@ static FUNC_NORETURN void drop_to_user(k_thread_entry_t user_entry,
void *p1, void *p2, void *p3)
{
u32_t stack_end;
struct z_x86_thread_stack_header *header =
(struct z_x86_thread_stack_header *)_current->stack_obj;
_current->arch.psp =
header->privilege_stack + sizeof(header->privilege_stack);
/* Transition will reset stack pointer to initial, discarding
* any old context since this is a one-way operation
@ -209,6 +211,9 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
} else
#endif /* CONFIG_X86_USERSPACE */
{
#ifdef CONFIG_X86_USERSPACE
thread->arch.ptables = &z_x86_kernel_ptables;
#endif
#ifdef _THREAD_WRAPPER_REQUIRED
initial_frame->edi = (u32_t)z_thread_entry;
initial_frame->thread_entry = z_x86_thread_entry_wrapper;

28
arch/x86/core/ia32/userspace.S
View file

@ -55,16 +55,10 @@ SECTION_FUNC(TEXT, z_x86_trampoline_to_kernel)
/* Save old trampoline stack pointer in %edi */
movl %esp, %edi
/* %esp = _kernel->current->stack_info.start
*
* This is the lowest address of the user mode stack, the PDPT is
* immediately before it, and then the highest address of the kernel
* stack. We want to transplant context here.
*/
/* Switch to privilege mode stack */
movl $_kernel, %esi
movl _kernel_offset_to_current(%esi), %esi
movl _thread_offset_to_stack_start(%esi), %esp
subl $Z_X86_PDPT_SIZE, %esp
movl _thread_offset_to_psp(%esi), %esp
/* Transplant stack context and restore ESI/EDI. Taking care to zero
* or put uninteresting values where we stashed ESI/EDI since the
@ -134,15 +128,11 @@ SECTION_FUNC(TEXT, z_x86_trampoline_to_user_always)
xchgl %edi, (%edi) /* Exchange old edi to restore it and put
trampoline stack address in its old storage
area */
/* Switch to user page table. The per-thread user page table is
* located at the highest addresses of the privilege mode elevation
* stack, immediately below the thread stack buffer.
*/
/* Switch to user page table */
pushl %eax
movl $_kernel, %eax
movl _kernel_offset_to_current(%eax), %eax
movl _thread_offset_to_stack_start(%eax), %eax
subl $Z_X86_PDPT_SIZE, %eax
movl _thread_offset_to_ptables(%eax), %eax
movl %eax, %cr3
popl %eax
movl $0, -4(%esp) /* Delete stashed EAX data */
@ -170,16 +160,10 @@ SECTION_FUNC(TEXT, z_x86_syscall_entry_stub)
/* Save old trampoline stack pointer in %edi */
movl %esp, %edi
/* %esp = _kernel->current->stack_info.start
*
* This is the lowest address of the user mode stack, the PDPT is
* immediately before it, and then the highest address of the kernel
* stack. We want to transplant context here.
*/
/* Switch to privilege elevation stack */
movl $_kernel, %esi
movl _kernel_offset_to_current(%esi), %esi
movl _thread_offset_to_stack_start(%esi), %esp
subl $Z_X86_PDPT_SIZE, %esp
movl _thread_offset_to_psp(%esi), %esp
/* Transplant context according to layout above. Variant of logic
* in x86_trampoline_to_kernel */

2
arch/x86/core/offsets/ia32_offsets.c
View file

@ -31,6 +31,8 @@ GEN_OFFSET_SYM(_thread_arch_t, excNestCount);
#endif
#ifdef CONFIG_USERSPACE
GEN_OFFSET_SYM(_thread_arch_t, psp);
GEN_OFFSET_SYM(_thread_arch_t, ptables);
GEN_ABSOLUTE_SYM(Z_X86_PDPT_SIZE, sizeof(struct x86_mmu_pdpt));
#endif

19
arch/x86/core/x86_mmu.c
View file

@ -990,11 +990,10 @@ static uintptr_t thread_pt_create(uintptr_t pages,
* | ... |
*/
static void copy_page_tables(struct k_thread *thread,
struct x86_page_tables *thread_ptables,
struct x86_page_tables *master_ptables)
{
uintptr_t pos, start;
struct x86_page_tables *thread_ptables =
z_x86_thread_page_tables_get(thread);
struct z_x86_thread_stack_header *header =
(struct z_x86_thread_stack_header *)thread->stack_obj;
@ -1091,11 +1090,21 @@ void z_x86_apply_mem_domain(struct x86_page_tables *ptables,
* user mode.
*
* Sets up the per-thread page tables, such that when they are activated on
* context switch, everything is ready to go.
* context switch, everything is ready to go. thread->arch.ptables is updated
* to the thread-level tables instead of the kernel's page tbales.
*/
void z_x86_thread_pt_init(struct k_thread *thread)
{
struct x86_page_tables *ptables = z_x86_thread_page_tables_get(thread);
struct x86_page_tables *ptables;
struct z_x86_thread_stack_header *header =
(struct z_x86_thread_stack_header *)thread->stack_obj;
#ifdef CONFIG_X86_64
ptables = (struct x86_page_tables *)(&header->page_tables);
#else
ptables = &header->kernel_data.ptables;
#endif
thread->arch.ptables = ptables;
/* USER_PDPT contains the page tables with the boot time memory
* policy. We use it as a template to set up the per-thread page
@ -1106,7 +1115,7 @@ void z_x86_thread_pt_init(struct k_thread *thread)
* accessible pages except the trampoline page marked as non-present.
* Without KPTI, they are the same object.
*/
copy_page_tables(thread, &USER_PTABLES);
copy_page_tables(thread, ptables, &USER_PTABLES);
/* Enable access to the thread's own stack buffer */
z_x86_mmu_set_flags(ptables, (void *)thread->stack_info.start,

8
arch/x86/include/ia32/kernel_arch_func.h
View file

@ -44,14 +44,6 @@ void z_x86_thread_pt_init(struct k_thread *thread);
void z_x86_apply_mem_domain(struct x86_page_tables *ptables,
struct k_mem_domain *mem_domain);
static inline struct x86_page_tables *
z_x86_thread_page_tables_get(struct k_thread *thread)
{
struct z_x86_thread_stack_header *header =
(struct z_x86_thread_stack_header *)thread->stack_obj;
return &header->kernel_data.ptables;
}
#endif /* CONFIG_USERSPACE */
/* ASM code to fiddle with registers to enable the MMU with PAE paging */

11
arch/x86/include/kernel_arch_func.h
View file

@ -7,6 +7,7 @@
#define ZEPHYR_ARCH_X86_INCLUDE_KERNEL_ARCH_FUNC_H_
#include <kernel_arch_data.h>
#include <arch/x86/mmustructs.h>
#ifdef CONFIG_X86_64
#include <intel64/kernel_arch_func.h>
@ -45,6 +46,16 @@ void z_x86_early_serial_init(void);
#ifdef CONFIG_X86_MMU
/* Create all page tables with boot configuration and enable paging */
void z_x86_paging_init(void);
static inline struct x86_page_tables *
z_x86_thread_page_tables_get(struct k_thread *thread)
{
#ifdef CONFIG_USERSPACE
return thread->arch.ptables;
#else
return &z_x86_kernel_ptables;
#endif
}
#endif /* CONFIG_X86_MMU */
/* Called upon CPU exception that is unhandled and hence fatal; dump

8
arch/x86/include/offsets_short_arch.h
View file

@ -15,4 +15,12 @@
#define _thread_offset_to_flags \
(___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET)
#ifdef CONFIG_USERSPACE
#define _thread_offset_to_psp \
(___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET)
#define _thread_offset_to_ptables \
(___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET)
#endif /* CONFIG_USERSPACE */
#endif /* ZEPHYR_ARCH_X86_INCLUDE_OFFSETS_SHORT_ARCH_H_ */

13
include/arch/x86/ia32/thread.h
View file

@ -208,6 +208,19 @@ typedef struct s_preempFloatReg {
struct _thread_arch {
u8_t flags;
#ifdef CONFIG_USERSPACE
/* Pointer to page tables used by this thread. Supervisor threads
* always use the kernel's page table, user thread use per-thread
* tables stored in the stack object
*/
struct x86_page_tables *ptables;
/* Initial privilege mode stack pointer when doing a system call.
* Un-set for supervisor threads.
*/
char *psp;
#endif
#if defined(CONFIG_LAZY_FP_SHARING)
/*
* Nested exception count to maintain setting of EXC_ACTIVE flag across