diff --git a/arch/arm/core/aarch64/Kconfig b/arch/arm/core/aarch64/Kconfig index 7e5758d0a82..97eaa73f4ca 100644 --- a/arch/arm/core/aarch64/Kconfig +++ b/arch/arm/core/aarch64/Kconfig @@ -64,6 +64,9 @@ config AARCH64_IMAGE_HEADER This option enables standard ARM64 boot image header used by Linux and understood by loaders such as u-boot on Xen xl tool. +config PRIVILEGED_STACK_SIZE + default 4096 + if CPU_CORTEX_A config ARMV8_A_NS diff --git a/arch/arm/core/aarch64/switch.S b/arch/arm/core/aarch64/switch.S index 1e5d3c441af..846516bf720 100644 --- a/arch/arm/core/aarch64/switch.S +++ b/arch/arm/core/aarch64/switch.S @@ -111,6 +111,11 @@ SECTION_FUNC(TEXT, z_arm64_sync_exc) cmp x1, #_SVC_CALL_RUNTIME_EXCEPT beq oops +#ifdef CONFIG_USERSPACE + cmp x1, #_SVC_CALL_SYSTEM_CALL + beq z_arm64_do_syscall +#endif + #ifdef CONFIG_IRQ_OFFLOAD cmp x1, #_SVC_CALL_IRQ_OFFLOAD beq offload diff --git a/arch/arm/core/aarch64/thread.c b/arch/arm/core/aarch64/thread.c index 37576bf79a0..66b3863fa61 100644 --- a/arch/arm/core/aarch64/thread.c +++ b/arch/arm/core/aarch64/thread.c @@ -54,6 +54,7 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, } pInitCtx->tpidrro_el0 = 0x0; + thread->arch.priv_stack_start = 0; #else pInitCtx->elr = (uint64_t)z_thread_entry; #endif @@ -82,6 +83,9 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, z_arch_esf_t *pInitCtx; uintptr_t stack_ptr; + /* Setup the private stack */ + _current->arch.priv_stack_start = (uint64_t)(_current->stack_obj); + /* Reset the stack pointer to the base discarding any old context */ stack_ptr = Z_STACK_PTR_ALIGN(_current->stack_info.start + _current->stack_info.size - diff --git a/arch/arm/core/aarch64/userspace.S b/arch/arm/core/aarch64/userspace.S index f24801fd54d..d58beb6aa18 100644 --- a/arch/arm/core/aarch64/userspace.S +++ b/arch/arm/core/aarch64/userspace.S @@ -85,6 +85,61 @@ abv_fail: mov x0, #-1 ret +/* + * System call entry point. + */ + +GTEXT(z_arm64_do_syscall) +SECTION_FUNC(TEXT, z_arm64_do_syscall) + /* Recover the syscall parameters from the ESF */ + ldp x0, x1, [sp, ___esf_t_x0_x1_OFFSET] + ldp x2, x3, [sp, ___esf_t_x2_x3_OFFSET] + ldp x4, x5, [sp, ___esf_t_x4_x5_OFFSET] + + /* Recover the syscall ID */ + ldr x8, [sp, ___esf_t_x8_x9_OFFSET] + + /* Check whether the ID is valid */ + ldr x9, =K_SYSCALL_LIMIT + cmp x8, x9 + blo valid_syscall_id + ldr x8, =K_SYSCALL_BAD + +valid_syscall_id: + ldr x9, =_k_syscall_table + ldr x9, [x9, x8, lsl #3] + + /* Recover the privileged stack */ +#ifdef CONFIG_SMP + get_cpu x10, x8 + ldr x10, [x10, #___cpu_t_current_OFFSET] +#else + ldr x10, =_kernel + ldr x10, [x10, #_kernel_offset_to_current] +#endif + ldr x10, [x10, #_thread_offset_to_priv_stack_start] + add x10, x10, #CONFIG_PRIVILEGED_STACK_SIZE + + /* Save the original SP on the privileged stack */ + mov x11, sp + mov sp, x10 + str x11, [sp, #-16]! + + /* Jump into the syscall */ + msr daifclr, #(DAIFSET_IRQ_BIT) + blr x9 + msr daifset, #(DAIFSET_IRQ_BIT) + + /* Restore the original SP containing the ESF */ + ldr x11, [sp], #16 + mov sp, x11 + + /* Save the return value into the ESF */ + str x0, [sp, ___esf_t_x0_x1_OFFSET] + + /* Return from exception */ + b z_arm64_exit_exc + /* * Routine to jump into userspace * diff --git a/arch/arm/core/offsets/offsets_aarch64.c b/arch/arm/core/offsets/offsets_aarch64.c index fc986be4949..5e17743b932 100644 --- a/arch/arm/core/offsets/offsets_aarch64.c +++ b/arch/arm/core/offsets/offsets_aarch64.c @@ -29,6 +29,10 @@ #include #include +#ifdef CONFIG_USERSPACE +GEN_OFFSET_SYM(_thread_arch_t, priv_stack_start); +#endif + GEN_NAMED_OFFSET_SYM(_callee_saved_t, x19, x19_x20); GEN_NAMED_OFFSET_SYM(_callee_saved_t, x21, x21_x22); GEN_NAMED_OFFSET_SYM(_callee_saved_t, x23, x23_x24); diff --git a/arch/arm/include/aarch64/offsets_short_arch.h b/arch/arm/include/aarch64/offsets_short_arch.h index 3b519f109a5..11bbb1fcd05 100644 --- a/arch/arm/include/aarch64/offsets_short_arch.h +++ b/arch/arm/include/aarch64/offsets_short_arch.h @@ -9,4 +9,9 @@ #include +#ifdef CONFIG_USERSPACE +#define _thread_offset_to_priv_stack_start \ + (___thread_t_arch_OFFSET + ___thread_arch_t_priv_stack_start_OFFSET) +#endif + #endif /* ZEPHYR_ARCH_ARM_INCLUDE_AARCH64_OFFSETS_SHORT_ARCH_H_ */ diff --git a/include/arch/arm/aarch64/syscall.h b/include/arch/arm/aarch64/syscall.h index 784fda80b40..c008a83207c 100644 --- a/include/arch/arm/aarch64/syscall.h +++ b/include/arch/arm/aarch64/syscall.h @@ -19,6 +19,7 @@ #define _SVC_CALL_CONTEXT_SWITCH 0 #define _SVC_CALL_IRQ_OFFLOAD 1 #define _SVC_CALL_RUNTIME_EXCEPT 2 +#define _SVC_CALL_SYSTEM_CALL 3 #ifdef CONFIG_USERSPACE #ifndef _ASMLANGUAGE @@ -31,6 +32,137 @@ extern "C" { #endif +/* + * Syscall invocation macros. arm-specific machine constraints used to ensure + * args land in the proper registers. + */ +static inline uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ + register uint64_t ret __asm__("x0") = arg1; + register uint64_t r1 __asm__("x1") = arg2; + register uint64_t r2 __asm__("x2") = arg3; + register uint64_t r3 __asm__("x3") = arg4; + register uint64_t r4 __asm__("x4") = arg5; + register uint64_t r5 __asm__("x5") = arg6; + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r1), "r" (r2), "r" (r3), + "r" (r4), "r" (r5), "r" (r8) + : "memory"); + + return ret; +} + +static inline uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, + uintptr_t call_id) +{ + register uint64_t ret __asm__("x0") = arg1; + register uint64_t r1 __asm__("x1") = arg2; + register uint64_t r2 __asm__("x2") = arg3; + register uint64_t r3 __asm__("x3") = arg4; + register uint64_t r4 __asm__("x4") = arg5; + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r1), "r" (r2), "r" (r3), + "r" (r4), "r" (r8) + : "memory"); + + return ret; +} + +static inline uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id) +{ + register uint64_t ret __asm__("x0") = arg1; + register uint64_t r1 __asm__("x1") = arg2; + register uint64_t r2 __asm__("x2") = arg3; + register uint64_t r3 __asm__("x3") = arg4; + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r1), "r" (r2), "r" (r3), + "r" (r8) + : "memory"); + + return ret; +} + +static inline uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, + uintptr_t call_id) +{ + register uint64_t ret __asm__("x0") = arg1; + register uint64_t r1 __asm__("x1") = arg2; + register uint64_t r2 __asm__("x2") = arg3; + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r1), "r" (r2), "r" (r8) + : "memory"); + + return ret; +} + +static inline uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id) +{ + register uint64_t ret __asm__("x0") = arg1; + register uint64_t r1 __asm__("x1") = arg2; + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r1), "r" (r8) + : "memory"); + + return ret; +} + +static inline uintptr_t arch_syscall_invoke1(uintptr_t arg1, + uintptr_t call_id) +{ + register uint64_t ret __asm__("x0") = arg1; + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r8) + : "memory"); + return ret; +} + +static inline uintptr_t arch_syscall_invoke0(uintptr_t call_id) +{ + register uint64_t ret __asm__("x0"); + register uint64_t r8 __asm__("x8") = call_id; + + __asm__ volatile("svc %[svid]\n" + : "=r"(ret) + : [svid] "i" (_SVC_CALL_SYSTEM_CALL), + "r" (ret), "r" (r8) + : "memory"); + + return ret; +} + static inline bool arch_is_user_context(void) { uint64_t tpidrro_el0; diff --git a/include/arch/arm/aarch64/thread.h b/include/arch/arm/aarch64/thread.h index 414245870c9..aecae07900c 100644 --- a/include/arch/arm/aarch64/thread.h +++ b/include/arch/arm/aarch64/thread.h @@ -40,7 +40,9 @@ struct _callee_saved { typedef struct _callee_saved _callee_saved_t; struct _thread_arch { - /* empty */ +#ifdef CONFIG_USERSPACE + uint64_t priv_stack_start; +#endif }; typedef struct _thread_arch _thread_arch_t;