x86: optimize locations of psp and thread ptables

z_x86_thread_page_tables_get() now works for both user
and supervisor threads, returning the kernel page tables
in the latter case. This API has been up-leveled to
a common header.

The per-thread privilege elevation stack initial stack
pointer, and the per-thread page table locations are no
longer computed from other values, and instead are stored
in thread->arch.

A problem where the wrong page tables were dumped out
on certain kinds of page faults has been fixed.

Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>

arch/x86/core/fatal.c

@@ -274,15 +274,7 @@ static void dump_page_fault(z_arch_esf_t *esf)
 	}
 
 #ifdef CONFIG_X86_MMU
-#ifdef CONFIG_USERSPACE
+	z_x86_dump_mmu_flags(z_x86_thread_page_tables_get(_current), cr2);
-	if (err & US) {
-		z_x86_dump_mmu_flags(z_x86_thread_page_tables_get(_current),
-				     cr2);
-	} else
-#endif /* CONFIG_USERSPACE */
-	{
-		z_x86_dump_mmu_flags(&z_x86_kernel_ptables, cr2);
-	}
 #endif /* CONFIG_X86_MMU */
 }
 #endif /* CONFIG_EXCEPTION_DEBUG */

arch/x86/core/ia32/thread.c

@@ -64,7 +64,8 @@ static inline void page_tables_set(struct x86_page_tables *ptables)
  */
 void z_x86_swap_update_page_tables(struct k_thread *incoming)
 {
-	struct x86_page_tables *ptables;
+	struct x86_page_tables *ptables =
+		z_x86_thread_page_tables_get(incoming);
 
 	/* If we're a user thread, we want the active page tables to
 	 * be the per-thread instance.
@@ -73,15 +74,11 @@ void z_x86_swap_update_page_tables(struct k_thread *incoming)
 	 * kernel page tables instead.
 	 */
 	if ((incoming->base.user_options & K_USER) != 0) {
-		ptables = z_x86_thread_page_tables_get(incoming);
-
 		/* In case of privilege elevation, use the incoming
 		 * thread's kernel stack. This area starts immediately
 		 * before the PDPT.
 		 */
-		_main_tss.esp0 = (uintptr_t)ptables;
+		_main_tss.esp0 = (uintptr_t)(incoming->arch.psp);
-	} else {
-		ptables = &z_x86_kernel_ptables;
 	}
 
 	/* Check first that we actually need to do this, since setting
@@ -97,6 +94,11 @@ static FUNC_NORETURN void drop_to_user(k_thread_entry_t user_entry,
 				       void *p1, void *p2, void *p3)
 {
 	u32_t stack_end;
+	struct z_x86_thread_stack_header *header =
+		(struct z_x86_thread_stack_header *)_current->stack_obj;
+
+	_current->arch.psp =
+		header->privilege_stack + sizeof(header->privilege_stack);
 
 	/* Transition will reset stack pointer to initial, discarding
 	 * any old context since this is a one-way operation
@@ -209,6 +211,9 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
 	} else
 #endif /* CONFIG_X86_USERSPACE */
 	{
+#ifdef CONFIG_X86_USERSPACE
+		thread->arch.ptables = &z_x86_kernel_ptables;
+#endif
 #ifdef _THREAD_WRAPPER_REQUIRED
 		initial_frame->edi = (u32_t)z_thread_entry;
 		initial_frame->thread_entry = z_x86_thread_entry_wrapper;

arch/x86/core/ia32/userspace.S

@@ -55,16 +55,10 @@ SECTION_FUNC(TEXT, z_x86_trampoline_to_kernel)
 	/* Save old trampoline stack pointer in %edi */
 	movl	%esp, %edi
 
-	/* %esp = _kernel->current->stack_info.start
+	/* Switch to privilege mode stack */
-	 *
-	 * This is the lowest address of the user mode stack, the PDPT is
-	 * immediately before it, and then the highest address of the kernel
-	 * stack. We want to transplant context here.
-	 */
 	movl	$_kernel, %esi
 	movl	_kernel_offset_to_current(%esi), %esi
-	movl	_thread_offset_to_stack_start(%esi), %esp
+	movl	_thread_offset_to_psp(%esi), %esp
-	subl	$Z_X86_PDPT_SIZE, %esp
 
 	/* Transplant stack context and restore ESI/EDI. Taking care to zero
 	 * or put uninteresting values where we stashed ESI/EDI since the
@@ -134,15 +128,11 @@ SECTION_FUNC(TEXT, z_x86_trampoline_to_user_always)
 	xchgl	%edi, (%edi)	/* Exchange old edi to restore it and put
 				   trampoline stack address in its old storage
 				   area */
-	/* Switch to user page table. The per-thread user page table is
+	/* Switch to user page table */
-	 * located at the highest addresses of the privilege mode elevation
-	 * stack, immediately below the thread stack buffer.
-	 */
 	pushl	%eax
 	movl	$_kernel, %eax
 	movl	_kernel_offset_to_current(%eax), %eax
-	movl	_thread_offset_to_stack_start(%eax), %eax
+	movl	_thread_offset_to_ptables(%eax), %eax
-	subl	$Z_X86_PDPT_SIZE, %eax
 	movl	%eax, %cr3
 	popl	%eax
 	movl	$0, -4(%esp)	/* Delete stashed EAX data */
@@ -170,16 +160,10 @@ SECTION_FUNC(TEXT, z_x86_syscall_entry_stub)
 	/* Save old trampoline stack pointer in %edi */
 	movl	%esp, %edi
 
-	/* %esp = _kernel->current->stack_info.start
+	/* Switch to privilege elevation stack */
-	 *
-	 * This is the lowest address of the user mode stack, the PDPT is
-	 * immediately before it, and then the highest address of the kernel
-	 * stack. We want to transplant context here.
-	 */
 	movl	$_kernel, %esi
 	movl	_kernel_offset_to_current(%esi), %esi
-	movl	_thread_offset_to_stack_start(%esi), %esp
+	movl	_thread_offset_to_psp(%esi), %esp
-	subl	$Z_X86_PDPT_SIZE, %esp
 
 	/* Transplant context according to layout above. Variant of logic
 	 * in x86_trampoline_to_kernel */

arch/x86/core/offsets/ia32_offsets.c

@@ -31,6 +31,8 @@ GEN_OFFSET_SYM(_thread_arch_t, excNestCount);
 #endif
 
 #ifdef CONFIG_USERSPACE
+GEN_OFFSET_SYM(_thread_arch_t, psp);
+GEN_OFFSET_SYM(_thread_arch_t, ptables);
 GEN_ABSOLUTE_SYM(Z_X86_PDPT_SIZE, sizeof(struct x86_mmu_pdpt));
 #endif
 

arch/x86/core/x86_mmu.c

@@ -990,11 +990,10 @@ static uintptr_t thread_pt_create(uintptr_t pages,
  * | ...                       |
  */
 static void copy_page_tables(struct k_thread *thread,
+			     struct x86_page_tables *thread_ptables,
 			     struct x86_page_tables *master_ptables)
 {
 	uintptr_t pos, start;
-	struct x86_page_tables *thread_ptables =
-		z_x86_thread_page_tables_get(thread);
 	struct z_x86_thread_stack_header *header =
 		(struct z_x86_thread_stack_header *)thread->stack_obj;
 
@@ -1091,11 +1090,21 @@ void z_x86_apply_mem_domain(struct x86_page_tables *ptables,
  * user mode.
  *
  * Sets up the per-thread page tables, such that when they are activated on
- * context switch, everything is ready to go.
+ * context switch, everything is ready to go. thread->arch.ptables is updated
+ * to the thread-level tables instead of the kernel's page tbales.
  */
 void z_x86_thread_pt_init(struct k_thread *thread)
 {
-	struct x86_page_tables *ptables = z_x86_thread_page_tables_get(thread);
+	struct x86_page_tables *ptables;
+	struct z_x86_thread_stack_header *header =
+		(struct z_x86_thread_stack_header *)thread->stack_obj;
+
+#ifdef CONFIG_X86_64
+	ptables = (struct x86_page_tables *)(&header->page_tables);
+#else
+	ptables = &header->kernel_data.ptables;
+#endif
+	thread->arch.ptables = ptables;
 
 	/* USER_PDPT contains the page tables with the boot time memory
 	 * policy. We use it as a template to set up the per-thread page
@@ -1106,7 +1115,7 @@ void z_x86_thread_pt_init(struct k_thread *thread)
 	 * accessible pages except the trampoline page marked as non-present.
 	 * Without KPTI, they are the same object.
 	 */
-	copy_page_tables(thread, &USER_PTABLES);
+	copy_page_tables(thread, ptables, &USER_PTABLES);
 
 	/* Enable access to the thread's own stack buffer */
 	z_x86_mmu_set_flags(ptables, (void *)thread->stack_info.start,

arch/x86/include/ia32/kernel_arch_func.h

@@ -44,14 +44,6 @@ void z_x86_thread_pt_init(struct k_thread *thread);
 void z_x86_apply_mem_domain(struct x86_page_tables *ptables,
 			    struct k_mem_domain *mem_domain);
 
-static inline struct x86_page_tables *
-z_x86_thread_page_tables_get(struct k_thread *thread)
-{
-	struct z_x86_thread_stack_header *header =
-		(struct z_x86_thread_stack_header *)thread->stack_obj;
-
-	return &header->kernel_data.ptables;
-}
 #endif /* CONFIG_USERSPACE */
 
 /* ASM code to fiddle with registers to enable the MMU with PAE paging */

arch/x86/include/kernel_arch_func.h

@@ -7,6 +7,7 @@
 #define ZEPHYR_ARCH_X86_INCLUDE_KERNEL_ARCH_FUNC_H_
 
 #include <kernel_arch_data.h>
+#include <arch/x86/mmustructs.h>
 
 #ifdef CONFIG_X86_64
 #include <intel64/kernel_arch_func.h>
@@ -45,6 +46,16 @@ void z_x86_early_serial_init(void);
 #ifdef CONFIG_X86_MMU
 /* Create all page tables with boot configuration and enable paging */
 void z_x86_paging_init(void);
+
+static inline struct x86_page_tables *
+z_x86_thread_page_tables_get(struct k_thread *thread)
+{
+#ifdef CONFIG_USERSPACE
+	return thread->arch.ptables;
+#else
+	return &z_x86_kernel_ptables;
+#endif
+}
 #endif /* CONFIG_X86_MMU */
 
 /* Called upon CPU exception that is unhandled and hence fatal; dump

arch/x86/include/offsets_short_arch.h

@@ -15,4 +15,12 @@
 #define _thread_offset_to_flags \
 	(___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET)
 
+#ifdef CONFIG_USERSPACE
+#define _thread_offset_to_psp \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET)
+
+#define _thread_offset_to_ptables \
+	(___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET)
+#endif /* CONFIG_USERSPACE */
+
 #endif /* ZEPHYR_ARCH_X86_INCLUDE_OFFSETS_SHORT_ARCH_H_ */

include/arch/x86/ia32/thread.h

@@ -208,6 +208,19 @@ typedef struct s_preempFloatReg {
 struct _thread_arch {
 	u8_t flags;
 
+#ifdef CONFIG_USERSPACE
+	/* Pointer to page tables used by this thread. Supervisor threads
+	 * always use the kernel's page table, user thread use per-thread
+	 * tables stored in the stack object
+	 */
+	struct x86_page_tables *ptables;
+
+	/* Initial privilege mode stack pointer when doing a system call.
+	 * Un-set for supervisor threads.
+	 */
+	char *psp;
+#endif
+
 #if defined(CONFIG_LAZY_FP_SHARING)
 	/*
 	 * Nested exception count to maintain setting of EXC_ACTIVE flag across