2019-11-10 17:17:19 +01:00
|
|
|
/*
|
|
|
|
|
* Copyright (c) 2019 Carlo Caione <ccaione@baylibre.com>
|
|
|
|
|
*
|
|
|
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @file
|
|
|
|
|
* @brief New thread creation for ARM64 Cortex-A
|
|
|
|
|
*
|
|
|
|
|
* Core thread related primitives for the ARM64 Cortex-A
|
|
|
|
|
*/
|
|
|
|
|
|
2022-05-06 10:49:15 +02:00
|
|
|
#include <zephyr/kernel.h>
|
2019-11-10 17:17:19 +01:00
|
|
|
#include <ksched.h>
|
2022-05-06 10:49:15 +02:00
|
|
|
#include <zephyr/arch/cpu.h>
|
2019-11-10 17:17:19 +01:00
|
|
|
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
/*
|
|
|
|
|
* Note about stack usage:
|
|
|
|
|
*
|
|
|
|
|
* [ see also comments in include/arch/arm64/thread_stack.h ]
|
|
|
|
|
*
|
|
|
|
|
* - kernel threads are running in EL1 using SP_EL1 as stack pointer during
|
|
|
|
|
* normal execution and during exceptions. They are by definition already
|
|
|
|
|
* running in a privileged stack that is their own.
|
|
|
|
|
*
|
|
|
|
|
* - user threads are running in EL0 using SP_EL0 as stack pointer during
|
|
|
|
|
* normal execution. When at exception is taken or a syscall is called the
|
|
|
|
|
* stack pointer switches to SP_EL1 and the execution starts using the
|
|
|
|
|
* privileged portion of the user stack without touching SP_EL0. This portion
|
2023-01-20 04:46:31 +01:00
|
|
|
* is marked as not user accessible in the MMU/MPU.
|
|
|
|
|
*
|
|
|
|
|
* - a stack guard region will be added bellow the kernel stack when
|
|
|
|
|
* ARM64_STACK_PROTECTION is enabled. In this case, SP_EL0 will always point
|
|
|
|
|
* to the safe exception stack in the kernel space. For the kernel thread,
|
|
|
|
|
* SP_EL0 will not change always pointing to safe exception stack. For the
|
|
|
|
|
* userspace thread, SP_EL0 will switch from the user stack to the safe
|
|
|
|
|
* exception stack when entering the EL1 mode, and restore to the user stack
|
|
|
|
|
* when backing to userspace (EL0).
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
*
|
|
|
|
|
* Kernel threads:
|
|
|
|
|
*
|
2023-01-20 04:46:31 +01:00
|
|
|
* High memory addresses
|
|
|
|
|
*
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
* +---------------+ <- stack_ptr
|
|
|
|
|
* E | ESF |
|
|
|
|
|
* L |<<<<<<<<<<<<<<<| <- SP_EL1
|
|
|
|
|
* 1 | |
|
2023-01-20 04:46:31 +01:00
|
|
|
* +---------------+ <- stack limit
|
|
|
|
|
* | Stack guard | } Z_ARM64_STACK_GUARD_SIZE (protected by MMU/MPU)
|
|
|
|
|
* +---------------+ <- stack_obj
|
|
|
|
|
*
|
|
|
|
|
* Low Memory addresses
|
|
|
|
|
*
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
*
|
|
|
|
|
* User threads:
|
|
|
|
|
*
|
2023-01-20 04:46:31 +01:00
|
|
|
* High memory addresses
|
|
|
|
|
*
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
* +---------------+ <- stack_ptr
|
|
|
|
|
* E | |
|
|
|
|
|
* L |<<<<<<<<<<<<<<<| <- SP_EL0
|
|
|
|
|
* 0 | |
|
|
|
|
|
* +---------------+ ..............|
|
|
|
|
|
* E | ESF | | Privileged portion of the stack
|
|
|
|
|
* L +>>>>>>>>>>>>>>>+ <- SP_EL1 |_ used during exceptions and syscalls
|
|
|
|
|
* 1 | | | of size ARCH_THREAD_STACK_RESERVED
|
2023-01-20 04:46:31 +01:00
|
|
|
* +---------------+ <- stack limit|
|
|
|
|
|
* | Stack guard | } Z_ARM64_STACK_GUARD_SIZE (protected by MMU/MPU)
|
|
|
|
|
* +---------------+ <- stack_obj
|
|
|
|
|
*
|
|
|
|
|
* Low Memory addresses
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
*
|
2022-03-14 23:11:00 +01:00
|
|
|
* When a kernel thread switches to user mode the SP_EL0 and SP_EL1
|
|
|
|
|
* values are reset accordingly in arch_user_mode_enter().
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
*/
|
|
|
|
|
|
2020-11-25 14:44:21 +01:00
|
|
|
#ifdef CONFIG_USERSPACE
|
|
|
|
|
static bool is_user(struct k_thread *thread)
|
|
|
|
|
{
|
|
|
|
|
return (thread->base.user_options & K_USER) != 0;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2019-11-10 17:17:19 +01:00
|
|
|
void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
|
2020-04-23 22:55:56 +02:00
|
|
|
char *stack_ptr, k_thread_entry_t entry,
|
2020-04-23 20:32:08 +02:00
|
|
|
void *p1, void *p2, void *p3)
|
2019-11-10 17:17:19 +01:00
|
|
|
{
|
2022-03-14 18:51:40 +01:00
|
|
|
extern void z_arm64_exit_exc(void);
|
2020-11-11 15:19:24 +01:00
|
|
|
z_arch_esf_t *pInitCtx;
|
2019-11-10 17:17:19 +01:00
|
|
|
|
2023-09-16 09:26:45 +02:00
|
|
|
/*
|
|
|
|
|
* Clean the thread->arch to avoid unexpected behavior because the
|
|
|
|
|
* thread->arch might be dirty
|
|
|
|
|
*/
|
|
|
|
|
memset(&thread->arch, 0, sizeof(thread->arch));
|
|
|
|
|
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
/*
|
|
|
|
|
* The ESF is now hosted at the top of the stack. For user threads this
|
|
|
|
|
* is also fine because at this stage they are still running in EL1.
|
|
|
|
|
* The context will be relocated by arch_user_mode_enter() before
|
|
|
|
|
* dropping into EL0.
|
|
|
|
|
*/
|
|
|
|
|
|
2020-11-11 15:19:24 +01:00
|
|
|
pInitCtx = Z_STACK_PTR_TO_FRAME(struct __esf, stack_ptr);
|
2019-11-10 17:17:19 +01:00
|
|
|
|
2020-11-11 15:19:24 +01:00
|
|
|
pInitCtx->x0 = (uint64_t)entry;
|
|
|
|
|
pInitCtx->x1 = (uint64_t)p1;
|
|
|
|
|
pInitCtx->x2 = (uint64_t)p2;
|
|
|
|
|
pInitCtx->x3 = (uint64_t)p3;
|
2020-03-23 11:49:58 +01:00
|
|
|
|
|
|
|
|
/*
|
2020-09-02 18:13:18 +02:00
|
|
|
* - ELR_ELn: to be used by eret in z_arm64_exit_exc() to return
|
2020-04-23 20:32:08 +02:00
|
|
|
* to z_thread_entry() with entry in x0(entry_point) and the
|
|
|
|
|
* parameters already in place in x1(arg1), x2(arg2), x3(arg3).
|
2020-09-02 18:13:18 +02:00
|
|
|
* - SPSR_ELn: to enable IRQs (we are masking FIQs).
|
2020-03-23 11:49:58 +01:00
|
|
|
*/
|
2020-11-25 14:44:21 +01:00
|
|
|
#ifdef CONFIG_USERSPACE
|
|
|
|
|
/*
|
|
|
|
|
* If the new thread is a user thread we jump into
|
|
|
|
|
* arch_user_mode_enter() when still in EL1.
|
|
|
|
|
*/
|
|
|
|
|
if (is_user(thread)) {
|
|
|
|
|
pInitCtx->elr = (uint64_t)arch_user_mode_enter;
|
|
|
|
|
} else {
|
|
|
|
|
pInitCtx->elr = (uint64_t)z_thread_entry;
|
|
|
|
|
}
|
2023-01-19 15:02:44 +01:00
|
|
|
|
2020-11-25 14:44:21 +01:00
|
|
|
#else
|
2020-05-27 18:26:57 +02:00
|
|
|
pInitCtx->elr = (uint64_t)z_thread_entry;
|
2020-11-25 14:44:21 +01:00
|
|
|
#endif
|
2023-01-19 15:02:44 +01:00
|
|
|
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
/* Keep using SP_EL1 */
|
|
|
|
|
pInitCtx->spsr = SPSR_MODE_EL1H | DAIF_FIQ_BIT;
|
2019-11-10 17:17:19 +01:00
|
|
|
|
2021-04-13 06:42:00 +02:00
|
|
|
/* thread birth happens through the exception return path */
|
|
|
|
|
thread->arch.exception_depth = 1;
|
|
|
|
|
|
2019-11-10 17:17:19 +01:00
|
|
|
/*
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
* We are saving SP_EL1 to pop out entry and parameters when going
|
|
|
|
|
* through z_arm64_exit_exc(). For user threads the definitive location
|
2022-03-14 23:11:00 +01:00
|
|
|
* of SP_EL1 will be set in arch_user_mode_enter().
|
2019-11-10 17:17:19 +01:00
|
|
|
*/
|
arm64: Rework stack usage
The ARM64 port is currently using SP_EL0 for everything: kernel threads,
user threads and exceptions. In addition when taking an exception the
exception code is still using the thread SP without relying on any
interrupt stack.
If from one hand this makes the context switch really quick because the
thread context is already on the thread stack so we have only to save
one register (SP) for the whole context, on the other hand the major
limitation introduced by this choice is that if for some reason the
thread SP is corrupted or pointing to some unaccessible location (for
example in case of stack overflow), the exception code is unable to
recover or even deal with it.
The usual way of dealing with this kind of problems is to use a
dedicated interrupt stack on SP_EL1 when servicing the exceptions. The
real drawback of this is that, in case of context switch, all the
context must be copied from the shared interrupt stack into a
thread-specific stack or structure, so it is really slow.
We use here an hybrid approach, sacrificing a bit of stack space for a
quicker context switch. While nothing really changes for kernel threads,
for user threads we now use the privileged stack (already present to
service syscalls) as interrupt stack.
When an exception arrives the code now switches to use SP_EL1 that for
user threads is always pointing inside the privileged portion of the
stack of the current running thread. This achieves two things: (1)
isolate exceptions and syscall code to use a stack that is isolated,
privileged and not accessible to user threads and (2) the thread SP is
not touched at all during exceptions, so it can be invalid or corrupted
without any direct consequence.
Signed-off-by: Carlo Caione <ccaione@baylibre.com>
2021-04-21 20:14:14 +02:00
|
|
|
thread->callee_saved.sp_elx = (uint64_t)pInitCtx;
|
2022-03-14 18:51:40 +01:00
|
|
|
thread->callee_saved.lr = (uint64_t)z_arm64_exit_exc;
|
2020-08-04 12:52:16 +02:00
|
|
|
|
|
|
|
|
thread->switch_handle = thread;
|
2023-01-20 17:07:24 +01:00
|
|
|
#if defined(CONFIG_ARM64_STACK_PROTECTION)
|
|
|
|
|
thread->arch.stack_limit = (uint64_t)stack + Z_ARM64_STACK_GUARD_SIZE;
|
|
|
|
|
z_arm64_thread_mem_domains_init(thread);
|
|
|
|
|
#endif
|
2020-08-04 12:52:16 +02:00
|
|
|
}
|
|
|
|
|
|
2020-11-25 14:44:21 +01:00
|
|
|
#ifdef CONFIG_USERSPACE
|
|
|
|
|
FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
|
|
|
|
void *p1, void *p2, void *p3)
|
|
|
|
|
{
|
2022-03-14 23:11:00 +01:00
|
|
|
uintptr_t stack_el0, stack_el1;
|
|
|
|
|
uint64_t tmpreg;
|
2020-11-25 14:44:21 +01:00
|
|
|
|
2020-11-26 12:57:36 +01:00
|
|
|
/* Map the thread stack */
|
2021-07-20 04:14:17 +02:00
|
|
|
z_arm64_thread_mem_domains_init(_current);
|
2020-11-26 12:57:36 +01:00
|
|
|
|
2022-03-14 23:11:00 +01:00
|
|
|
/* Top of the user stack area */
|
|
|
|
|
stack_el0 = Z_STACK_PTR_ALIGN(_current->stack_info.start +
|
2020-11-25 14:44:21 +01:00
|
|
|
_current->stack_info.size -
|
|
|
|
|
_current->stack_info.delta);
|
|
|
|
|
|
2022-03-14 23:11:00 +01:00
|
|
|
/* Top of the privileged non-user-accessible part of the stack */
|
|
|
|
|
stack_el1 = (uintptr_t)(_current->stack_obj + ARCH_THREAD_STACK_RESERVED);
|
|
|
|
|
|
|
|
|
|
register void *x0 __asm__("x0") = user_entry;
|
|
|
|
|
register void *x1 __asm__("x1") = p1;
|
|
|
|
|
register void *x2 __asm__("x2") = p2;
|
|
|
|
|
register void *x3 __asm__("x3") = p3;
|
|
|
|
|
|
|
|
|
|
/* we don't want to be disturbed when playing with SPSR and ELR */
|
|
|
|
|
arch_irq_lock();
|
|
|
|
|
|
|
|
|
|
/* set up and drop into EL0 */
|
|
|
|
|
__asm__ volatile (
|
|
|
|
|
"mrs %[tmp], tpidrro_el0\n\t"
|
|
|
|
|
"orr %[tmp], %[tmp], %[is_usermode_flag]\n\t"
|
|
|
|
|
"msr tpidrro_el0, %[tmp]\n\t"
|
|
|
|
|
"msr elr_el1, %[elr]\n\t"
|
|
|
|
|
"msr spsr_el1, %[spsr]\n\t"
|
|
|
|
|
"msr sp_el0, %[sp_el0]\n\t"
|
|
|
|
|
"mov sp, %[sp_el1]\n\t"
|
|
|
|
|
"eret"
|
|
|
|
|
: [tmp] "=&r" (tmpreg)
|
|
|
|
|
: "r" (x0), "r" (x1), "r" (x2), "r" (x3),
|
|
|
|
|
[is_usermode_flag] "i" (TPIDRROEL0_IN_EL0),
|
|
|
|
|
[elr] "r" (z_thread_entry),
|
|
|
|
|
[spsr] "r" (DAIF_FIQ_BIT | SPSR_MODE_EL0T),
|
|
|
|
|
[sp_el0] "r" (stack_el0),
|
|
|
|
|
[sp_el1] "r" (stack_el1)
|
|
|
|
|
: "memory");
|
2020-11-25 14:44:21 +01:00
|
|
|
|
|
|
|
|
CODE_UNREACHABLE;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
