x86: de-couple user mode and HW stack protection

This is intended for memory-constrained systems and will save
4K per thread, since we will no longer reserve room for or
activate a kernel stack guard page.

If CONFIG_USERSPACE is enabled, stack overflows will still be
caught in some situations:

1) User mode threads overflowing stack, since it crashes into the
kernel stack page
2) Supervisor mode threads overflowing stack, since the kernel
stack page is marked non-present for non-user threads

Stack overflows will not be caught:

1) When handling a system call
2) When the interrupt stack overflows

Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>

arch/Kconfig

@@ -42,18 +42,30 @@ config HW_STACK_PROTECTION
 	bool "Hardware Stack Protection"
 	depends on ARCH_HAS_STACK_PROTECTION
 	help
-	  Select this option to enable hardware stack protection.
+	Select this option to enable hardware-based platform features to
+	catch stack overflows when the system is running in privileged
+	mode. If CONFIG_USERSPACE is not enabled, the system is always
+	running in privileged mode.
+
+	Note that this does not necessarily prevent corruptuon and assertions
+	about the overall system state when a fault is triggered cannot be
+	made.
 
 config USERSPACE
 	bool "User mode threads (EXPERIMENTAL)"
 	depends on ARCH_HAS_USERSPACE
-	select HW_STACK_PROTECTION
 	help
 	When enabled, threads may be created or dropped down to user mode,
 	which has significantly restricted permissions and must interact
 	with the kernel via system calls. See Zephyr documentation for more
 	details	about this feature.
 
+	If a user thread overflows its stack, this will be caught and the
+	kernel itself will be shielded from harm. Enabling this option
+	may or may not catch stack overflows when the system is in
+	privileged mode or handling a system call; to ensure these are always
+	caught, enable CONFIG_HW_STACK_PROTECTION.
+
 	This feature is under heavy development and APIs related to it are
 	subject to change, even if declared non-private.
 

arch/x86/Kconfig

@@ -96,11 +96,21 @@ config X86_PAE_MODE
 	Note: Do not enable in RAM constrained devices.
 endmenu
 
+config X86_ENABLE_TSS
+	bool
+	help
+	This hidden option enables defining a Task State Segment (TSS) for
+	kernel execution. This is needed to handle double-faults or
+	do privilege elevation. It also defines a special TSS and handler
+	for correctly handling double-fault exceptions, instead of just
+	letting the system triple-fault and reset.
+
 config X86_STACK_PROTECTION
 	bool
 	default y if HW_STACK_PROTECTION
 	select SET_GDT
 	select GDT_DYNAMIC
+	select X86_ENABLE_TSS
 	help
 	This option leverages the MMU to cause a system fatal error if the
 	bounds of the current process stack are overflowed. This is done
@@ -110,6 +120,9 @@ config X86_USERSPACE
 	bool
 	default y if USERSPACE
 	select THREAD_STACK_INFO
+	select SET_GDT
+	select GDT_DYNAMIC
+	select X86_ENABLE_TSS
 	help
 	This option enables APIs to drop a thread's privileges down to ring 3,
 	supporting user-level threads that are protected from each other and

arch/x86/core/crt0.S

@@ -375,7 +375,7 @@ __csSet:
 
 #endif /* CONFIG_X86_MMU */
 
-#ifdef CONFIG_X86_STACK_PROTECTION
+#if defined(CONFIG_X86_ENABLE_TSS)
 	mov $MAIN_TSS, %ax
 	ltr %ax
 #endif

arch/x86/core/fatal.c

@@ -65,7 +65,8 @@ FUNC_NORETURN void _NanoFatalErrorHandler(unsigned int reason,
 		break;
 	}
 #if defined(CONFIG_STACK_CANARIES) || defined(CONFIG_STACK_SENTINEL) || \
-		defined(CONFIG_X86_STACK_PROTECTION)
+		defined(CONFIG_HW_STACK_PROTECTION) || \
+		defined(CONFIG_USERSPACE)
 	case _NANO_ERR_STACK_CHK_FAIL:
 		printk("***** Stack Check Fail! *****\n");
 		break;
@@ -211,7 +212,7 @@ EXC_FUNC_NOCODE(IV_OVERFLOW);
 EXC_FUNC_NOCODE(IV_BOUND_RANGE);
 EXC_FUNC_NOCODE(IV_INVALID_OPCODE);
 EXC_FUNC_NOCODE(IV_DEVICE_NOT_AVAILABLE);
-#ifndef CONFIG_X86_STACK_PROTECTION
+#ifndef CONFIG_X86_ENABLE_TSS
 EXC_FUNC_NOCODE(IV_DOUBLE_FAULT);
 #endif
 EXC_FUNC_CODE(IV_INVALID_TSS);
@@ -294,7 +295,7 @@ FUNC_NORETURN void page_fault_handler(const NANO_ESF *pEsf)
 _EXCEPTION_CONNECT_CODE(page_fault_handler, IV_PAGE_FAULT);
 #endif /* CONFIG_EXCEPTION_DEBUG */
 
-#ifdef CONFIG_X86_STACK_PROTECTION
+#ifdef CONFIG_X86_ENABLE_TSS
 static __noinit volatile NANO_ESF _df_esf;
 
 /* Very tiny stack; just enough for the bogus error code pushed by the CPU
@@ -396,4 +397,4 @@ static FUNC_NORETURN __used void _df_handler_top(void)
  */
 _X86_IDT_TSS_REGISTER(DF_TSS, -1, -1, IV_DOUBLE_FAULT, 0);
 
-#endif /* CONFIG_X86_STACK_PROTECTION */
+#endif /* CONFIG_X86_ENABLE_TSS */

include/arch/x86/arch.h

@@ -552,10 +552,11 @@ extern FUNC_NORETURN void _SysFatalErrorHandler(unsigned int reason,
 						const NANO_ESF * pEsf);
 
 
-#ifdef CONFIG_X86_STACK_PROTECTION
+#ifdef CONFIG_X86_ENABLE_TSS
 extern struct task_state_segment _main_tss;
+#endif
 
-#ifdef CONFIG_X86_USERSPACE
+#ifdef CONFIG_USERSPACE
 /* Syscall invocation macros. x86-specific machine constraints used to ensure
  * args land in the proper registers, see implementation of
  * _x86_syscall_entry_stub in userspace.S
@@ -688,8 +689,12 @@ static inline int _arch_is_user_context(void)
 
 	return cs == USER_CODE_SEG;
 }
+#endif /* CONFIG_USERSPACE */
 
-/* With userspace enabled, stacks are arranged as follows:
+
+#if defined(CONFIG_HW_STACK_PROTECTION) && defined(CONFIG_USERSPACE)
+/* With both hardware stack protection and userspace enabled, stacks are
+ * arranged as follows:
  *
  * High memory addresses
  * +---------------+
@@ -711,14 +716,18 @@ static inline int _arch_is_user_context(void)
  * All context switches will save/restore the esp0 field in the TSS.
  */
 #define _STACK_GUARD_SIZE	(MMU_PAGE_SIZE * 2)
-#else /* !CONFIG_X86_USERSPACE */
-#define _STACK_GUARD_SIZE	MMU_PAGE_SIZE
-#endif /* CONFIG_X86_USERSPACE */
 #define _STACK_BASE_ALIGN	MMU_PAGE_SIZE
-#else /* !CONFIG_X86_STACK_PROTECTION */
+#elif defined(CONFIG_HW_STACK_PROTECTION) || defined(CONFIG_USERSPACE)
+/* If only one of HW stack protection or userspace is enabled, then the
+ * stack will be preceded by one page which is a guard page or a kernel mode
+ * stack, respectively.
+ */
+#define _STACK_GUARD_SIZE	MMU_PAGE_SIZE
+#define _STACK_BASE_ALIGN	MMU_PAGE_SIZE
+#else /* Neither feature */
 #define _STACK_GUARD_SIZE	0
 #define _STACK_BASE_ALIGN	STACK_ALIGN
-#endif /* CONFIG_X86_STACK_PROTECTION */
+#endif
 
 #define _ARCH_THREAD_STACK_DEFINE(sym, size) \
 	struct _k_thread_stack_element __kernel_noinit \