arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads, user threads and exceptions. In addition when taking an exception the exception code is still using the thread SP without relying on any interrupt stack. If from one hand this makes the context switch really quick because the thread context is already on the thread stack so we have only to save one register (SP) for the whole context, on the other hand the major limitation introduced by this choice is that if for some reason the thread SP is corrupted or pointing to some unaccessible location (for example in case of stack overflow), the exception code is unable to recover or even deal with it. The usual way of dealing with this kind of problems is to use a dedicated interrupt stack on SP_EL1 when servicing the exceptions. The real drawback of this is that, in case of context switch, all the context must be copied from the shared interrupt stack into a thread-specific stack or structure, so it is really slow. We use here an hybrid approach, sacrificing a bit of stack space for a quicker context switch. While nothing really changes for kernel threads, for user threads we now use the privileged stack (already present to service syscalls) as interrupt stack. When an exception arrives the code now switches to use SP_EL1 that for user threads is always pointing inside the privileged portion of the stack of the current running thread. This achieves two things: (1) isolate exceptions and syscall code to use a stack that is isolated, privileged and not accessible to user threads and (2) the thread SP is not touched at all during exceptions, so it can be invalid or corrupted without any direct consequence. Signed-off-by: Carlo Caione <ccaione@baylibre.com>
This commit is contained in:
parent
0ab51ff657
commit
256ca55476
11 changed files with 116 additions and 53 deletions
|
@ -30,16 +30,13 @@
|
|||
#include <kernel_arch_data.h>
|
||||
#include <kernel_offsets.h>
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
GEN_OFFSET_SYM(_thread_arch_t, priv_stack_start);
|
||||
#endif
|
||||
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x19, x19_x20);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x21, x21_x22);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x23, x23_x24);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x25, x25_x26);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x27, x27_x28);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x29, x29_sp);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, x29, x29_sp_el0);
|
||||
GEN_NAMED_OFFSET_SYM(_callee_saved_t, sp_elx, sp_elx);
|
||||
|
||||
GEN_ABSOLUTE_SYM(___callee_saved_t_SIZEOF, sizeof(struct _callee_saved));
|
||||
|
||||
|
|
|
@ -49,6 +49,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c)
|
|||
/* Custom plat prep_c init */
|
||||
bl z_arm64_el3_plat_prep_c
|
||||
|
||||
/* Set SP_EL1 */
|
||||
msr sp_el1, x24
|
||||
|
||||
b out
|
||||
2:
|
||||
/* Disable alignment fault checking */
|
||||
|
@ -59,6 +62,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c)
|
|||
/* Custom plat prep_c init */
|
||||
bl z_arm64_el2_plat_prep_c
|
||||
|
||||
/* Set SP_EL1 */
|
||||
msr sp_el1, x24
|
||||
|
||||
b out
|
||||
1:
|
||||
/* Disable alignment fault checking */
|
||||
|
@ -69,6 +75,9 @@ SECTION_SUBSEC_FUNC(TEXT,_reset_section,__reset_prep_c)
|
|||
/* Custom plat prep_c init */
|
||||
bl z_arm64_el1_plat_prep_c
|
||||
|
||||
/* Set SP_EL1. We cannot use sp_el1 at EL1 */
|
||||
msr SPSel, #1
|
||||
mov sp, x24
|
||||
out:
|
||||
isb
|
||||
|
||||
|
|
|
@ -35,15 +35,19 @@ SECTION_FUNC(TEXT, z_arm64_context_switch)
|
|||
/* addr of callee-saved regs in thread in x2 */
|
||||
add x2, x1, x3
|
||||
|
||||
/* Save the current SP */
|
||||
mov x4, sp
|
||||
/* Save the current SP_EL0 */
|
||||
mrs x4, sp_el0
|
||||
|
||||
stp x19, x20, [x2, ___callee_saved_t_x19_x20_OFFSET]
|
||||
stp x21, x22, [x2, ___callee_saved_t_x21_x22_OFFSET]
|
||||
stp x23, x24, [x2, ___callee_saved_t_x23_x24_OFFSET]
|
||||
stp x25, x26, [x2, ___callee_saved_t_x25_x26_OFFSET]
|
||||
stp x27, x28, [x2, ___callee_saved_t_x27_x28_OFFSET]
|
||||
stp x29, x4, [x2, ___callee_saved_t_x29_sp_OFFSET]
|
||||
stp x29, x4, [x2, ___callee_saved_t_x29_sp_el0_OFFSET]
|
||||
|
||||
/* Save the current SP_ELx */
|
||||
mov x4, sp
|
||||
stp x4, xzr, [x2, ___callee_saved_t_sp_elx_OFFSET]
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
/* save old thread into switch handle which is required by
|
||||
|
@ -72,8 +76,13 @@ SECTION_FUNC(TEXT, z_arm64_context_switch)
|
|||
ldp x23, x24, [x2, ___callee_saved_t_x23_x24_OFFSET]
|
||||
ldp x25, x26, [x2, ___callee_saved_t_x25_x26_OFFSET]
|
||||
ldp x27, x28, [x2, ___callee_saved_t_x27_x28_OFFSET]
|
||||
ldp x29, x1, [x2, ___callee_saved_t_x29_sp_OFFSET]
|
||||
ldp x29, x1, [x2, ___callee_saved_t_x29_sp_el0_OFFSET]
|
||||
|
||||
/* Restore SP_EL0 */
|
||||
msr sp_el0, x1
|
||||
|
||||
/* Restore SP_EL1 */
|
||||
ldp x1, xzr, [x2, ___callee_saved_t_sp_elx_OFFSET]
|
||||
mov sp, x1
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
|
|
|
@ -16,6 +16,49 @@
|
|||
#include <wait_q.h>
|
||||
#include <arch/cpu.h>
|
||||
|
||||
/*
|
||||
* Note about stack usage:
|
||||
*
|
||||
* [ see also comments in include/arch/arm64/thread_stack.h ]
|
||||
*
|
||||
* - kernel threads are running in EL1 using SP_EL1 as stack pointer during
|
||||
* normal execution and during exceptions. They are by definition already
|
||||
* running in a privileged stack that is their own.
|
||||
*
|
||||
* - user threads are running in EL0 using SP_EL0 as stack pointer during
|
||||
* normal execution. When at exception is taken or a syscall is called the
|
||||
* stack pointer switches to SP_EL1 and the execution starts using the
|
||||
* privileged portion of the user stack without touching SP_EL0. This portion
|
||||
* is marked as not user accessible in the MMU.
|
||||
*
|
||||
* Kernel threads:
|
||||
*
|
||||
* +---------------+ <- stack_ptr
|
||||
* E | ESF |
|
||||
* L |<<<<<<<<<<<<<<<| <- SP_EL1
|
||||
* 1 | |
|
||||
* +---------------+
|
||||
|
||||
*
|
||||
* User threads:
|
||||
*
|
||||
* +---------------+ <- stack_ptr
|
||||
* E | |
|
||||
* L |<<<<<<<<<<<<<<<| <- SP_EL0
|
||||
* 0 | |
|
||||
* +---------------+ ..............|
|
||||
* E | ESF | | Privileged portion of the stack
|
||||
* L +>>>>>>>>>>>>>>>+ <- SP_EL1 |_ used during exceptions and syscalls
|
||||
* 1 | | | of size ARCH_THREAD_STACK_RESERVED
|
||||
* +---------------+ <- stack_obj..|
|
||||
*
|
||||
* When a new user thread is created or when a kernel thread switches to user
|
||||
* mode the initial ESF is relocated to the privileged portion of the stack
|
||||
* and the values of stack_ptr, SP_EL0 and SP_EL1 are correctly reset when
|
||||
* going through arch_user_mode_enter() and z_arm64_userspace_enter()
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
static bool is_user(struct k_thread *thread)
|
||||
{
|
||||
|
@ -29,6 +72,13 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
|
|||
{
|
||||
z_arch_esf_t *pInitCtx;
|
||||
|
||||
/*
|
||||
* The ESF is now hosted at the top of the stack. For user threads this
|
||||
* is also fine because at this stage they are still running in EL1.
|
||||
* The context will be relocated by arch_user_mode_enter() before
|
||||
* dropping into EL0.
|
||||
*/
|
||||
|
||||
pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr);
|
||||
|
||||
pInitCtx->x0 = (uint64_t)entry;
|
||||
|
@ -52,18 +102,19 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
|
|||
} else {
|
||||
pInitCtx->elr = (uint64_t)z_thread_entry;
|
||||
}
|
||||
|
||||
thread->arch.priv_stack_start = 0;
|
||||
#else
|
||||
pInitCtx->elr = (uint64_t)z_thread_entry;
|
||||
#endif
|
||||
pInitCtx->spsr = SPSR_MODE_EL1T | DAIF_FIQ_BIT;
|
||||
/* Keep using SP_EL1 */
|
||||
pInitCtx->spsr = SPSR_MODE_EL1H | DAIF_FIQ_BIT;
|
||||
|
||||
/*
|
||||
* We are saving SP to pop out entry and parameters when going through
|
||||
* z_arm64_exit_exc()
|
||||
* We are saving SP_EL1 to pop out entry and parameters when going
|
||||
* through z_arm64_exit_exc(). For user threads the definitive location
|
||||
* of SP_EL1 will be set implicitly when going through
|
||||
* z_arm64_userspace_enter() (see comments there)
|
||||
*/
|
||||
thread->callee_saved.sp = (uint64_t)pInitCtx;
|
||||
thread->callee_saved.sp_elx = (uint64_t)pInitCtx;
|
||||
|
||||
thread->switch_handle = thread;
|
||||
}
|
||||
|
@ -85,20 +136,24 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
|||
/* Map the thread stack */
|
||||
z_arm64_thread_pt_init(_current);
|
||||
|
||||
/* Setup the private stack */
|
||||
_current->arch.priv_stack_start = (uint64_t)(_current->stack_obj);
|
||||
|
||||
/* Reset the stack pointer to the base discarding any old context */
|
||||
/*
|
||||
* Reset the SP_EL0 stack pointer to the stack top discarding any old
|
||||
* context. The actual register is written in z_arm64_userspace_enter()
|
||||
*/
|
||||
stack_ptr = Z_STACK_PTR_ALIGN(_current->stack_info.start +
|
||||
_current->stack_info.size -
|
||||
_current->stack_info.delta);
|
||||
|
||||
/*
|
||||
* Reconstruct the ESF from scratch to leverage the z_arm64_exit_exc()
|
||||
* macro that will simulate a return from exception to move from EL1t
|
||||
* to EL0t. On return we will be in userspace.
|
||||
* macro that will simulate a return from exception to move from EL1h
|
||||
* to EL0t. On return we will be in userspace using SP_EL0.
|
||||
*
|
||||
* We relocate the ESF to the beginning of the privileged stack in the
|
||||
* not user accessible part of the stack
|
||||
*/
|
||||
pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr);
|
||||
pInitCtx = (struct __esf *) (_current->stack_obj + ARCH_THREAD_STACK_RESERVED -
|
||||
sizeof(struct __esf));
|
||||
|
||||
pInitCtx->spsr = DAIF_FIQ_BIT | SPSR_MODE_EL0T;
|
||||
pInitCtx->elr = (uint64_t)z_thread_entry;
|
||||
|
@ -109,7 +164,7 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
|||
pInitCtx->x3 = (uint64_t)p3;
|
||||
|
||||
/* All the needed information is already in the ESF */
|
||||
z_arm64_userspace_enter(pInitCtx);
|
||||
z_arm64_userspace_enter(pInitCtx, stack_ptr);
|
||||
|
||||
CODE_UNREACHABLE;
|
||||
}
|
||||
|
|
|
@ -115,26 +115,11 @@ valid_syscall_id:
|
|||
ldr x9, =_k_syscall_table
|
||||
ldr x9, [x9, x8, lsl #3]
|
||||
|
||||
/* Recover the privileged stack */
|
||||
get_cpu x10
|
||||
ldr x10, [x10, #___cpu_t_current_OFFSET]
|
||||
ldr x10, [x10, #_thread_offset_to_priv_stack_start]
|
||||
add x10, x10, #CONFIG_PRIVILEGED_STACK_SIZE
|
||||
|
||||
/* Save the original SP on the privileged stack */
|
||||
mov x11, sp
|
||||
mov sp, x10
|
||||
str x11, [sp, #-16]!
|
||||
|
||||
/* Jump into the syscall */
|
||||
msr daifclr, #(DAIFSET_IRQ_BIT)
|
||||
blr x9
|
||||
msr daifset, #(DAIFSET_IRQ_BIT)
|
||||
|
||||
/* Restore the original SP containing the ESF */
|
||||
ldr x11, [sp], #16
|
||||
mov sp, x11
|
||||
|
||||
/* Save the return value into the ESF */
|
||||
str x0, [sp, ___esf_t_x0_x1_OFFSET]
|
||||
|
||||
|
@ -151,5 +136,18 @@ valid_syscall_id:
|
|||
|
||||
GTEXT(z_arm64_userspace_enter)
|
||||
SECTION_FUNC(TEXT, z_arm64_userspace_enter)
|
||||
/*
|
||||
* When a kernel thread is moved to user mode it doesn't have any
|
||||
* SP_EL0 set yet. We set it here for the first time pointing to the
|
||||
* beginning of the user accessible part of the stack (the top).
|
||||
*/
|
||||
msr sp_el0, x1
|
||||
|
||||
/*
|
||||
* Set SP_EL1 to point at the end of the ESF. Since we have relocated
|
||||
* the ESF at the beginning of the privileged stack area, when the ESF
|
||||
* is popped out by z_arm64_exit_exc() the SP_EL1 will be at the right
|
||||
* location for when the next exception will come.
|
||||
*/
|
||||
mov sp, x0
|
||||
b z_arm64_exit_exc
|
||||
|
|
|
@ -25,10 +25,6 @@ _ASM_FILE_PROLOGUE
|
|||
*/
|
||||
|
||||
.macro z_arm64_enter_exc xreg0, xreg1
|
||||
|
||||
/* Switch to SP_EL0 */
|
||||
msr spsel, #0
|
||||
|
||||
/*
|
||||
* Two things can happen to the remaining registers:
|
||||
*
|
||||
|
|
|
@ -40,7 +40,7 @@ static inline void arch_switch(void *switch_to, void **switched_from)
|
|||
}
|
||||
|
||||
extern void z_arm64_fatal_error(z_arch_esf_t *esf, unsigned int reason);
|
||||
extern void z_arm64_userspace_enter(z_arch_esf_t *esf);
|
||||
extern void z_arm64_userspace_enter(z_arch_esf_t *esf, uintptr_t sp_el0);
|
||||
extern void z_arm64_set_ttbr0(uintptr_t ttbr0);
|
||||
extern void z_arm64_ptable_ipi(void);
|
||||
|
||||
|
|
|
@ -9,9 +9,4 @@
|
|||
|
||||
#include <offsets.h>
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
#define _thread_offset_to_priv_stack_start \
|
||||
(___thread_t_arch_OFFSET + ___thread_arch_t_priv_stack_start_OFFSET)
|
||||
#endif
|
||||
|
||||
#endif /* ZEPHYR_ARCH_ARM64_INCLUDE_OFFSETS_SHORT_ARCH_H_ */
|
||||
|
|
|
@ -33,8 +33,10 @@ struct _callee_saved {
|
|||
uint64_t x26;
|
||||
uint64_t x27;
|
||||
uint64_t x28;
|
||||
uint64_t x29; /* FP */
|
||||
uint64_t sp;
|
||||
uint64_t x29;
|
||||
uint64_t sp_el0;
|
||||
uint64_t sp_elx;
|
||||
uint64_t xzr;
|
||||
};
|
||||
|
||||
typedef struct _callee_saved _callee_saved_t;
|
||||
|
@ -42,7 +44,6 @@ typedef struct _callee_saved _callee_saved_t;
|
|||
struct _thread_arch {
|
||||
#ifdef CONFIG_USERSPACE
|
||||
struct arm_mmu_ptables *ptables;
|
||||
uint64_t priv_stack_start;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
|
|
@ -18,6 +18,8 @@
|
|||
#endif
|
||||
|
||||
/*
|
||||
* [ see also comments in arch/arm64/core/thread.c ]
|
||||
*
|
||||
* High memory addresses
|
||||
*
|
||||
* +-------------------+ <- thread.stack_info.start + thread.stack_info.size
|
||||
|
@ -31,7 +33,7 @@
|
|||
* | Unused stack |
|
||||
* | |
|
||||
* +-------------------+ <- thread.stack_info.start
|
||||
* | Reserved memory | } K_(THREAD|KERNEL)_STACK_RESERVED
|
||||
* | Privileged stack | } K_(THREAD|KERNEL)_STACK_RESERVED
|
||||
* +-------------------+ <- thread.stack_obj
|
||||
*
|
||||
* Low Memory addresses
|
||||
|
|
|
@ -51,8 +51,9 @@ size_t _kernel_thread_info_offsets[] = {
|
|||
user_options),
|
||||
[THREAD_INFO_OFFSET_T_PRIO] = offsetof(struct _thread_base, prio),
|
||||
#if defined(CONFIG_ARM64)
|
||||
/* We are assuming that the SP of interest is SP_EL1 */
|
||||
[THREAD_INFO_OFFSET_T_STACK_PTR] = offsetof(struct k_thread,
|
||||
callee_saved.sp),
|
||||
callee_saved.sp_elx),
|
||||
#elif defined(CONFIG_ARM)
|
||||
[THREAD_INFO_OFFSET_T_STACK_PTR] = offsetof(struct k_thread,
|
||||
callee_saved.psp),
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue