diff --git a/arch/Kconfig b/arch/Kconfig index 5e3b96f414d..cc9925e7774 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -125,6 +125,7 @@ config XTENSA select IRQ_OFFLOAD_NESTED if IRQ_OFFLOAD select ARCH_HAS_CODE_DATA_RELOCATION select ARCH_HAS_TIMING_FUNCTIONS + select ARCH_MEM_DOMAIN_DATA if USERSPACE imply ATOMIC_OPERATIONS_ARCH help Xtensa architecture diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index a1517f17ed0..c875aa30972 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -113,6 +113,7 @@ config XTENSA_MMU bool "Xtensa MMU Support" default n select MMU + select ARCH_MEM_DOMAIN_SYNCHRONOUS_API if USERSPACE select XTENSA_SMALL_VECTOR_TABLE_ENTRY select KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK if XTENSA_RPO_CACHE help @@ -144,8 +145,18 @@ if XTENSA_MMU The bit shift number for the virtual address for Xtensa page table (PTEVADDR). + config XTENSA_MMU_NUM_L1_TABLES + int "Number of L1 page tables" + default 1 if !USERSPACE + default 4 + help + This option specifies the maximum number of traslation tables. + Translation tables are directly related to the number of + memory domains in the target, considering the kernel itself requires one. + config XTENSA_MMU_NUM_L2_TABLES int "Number of L2 page tables" + default 20 if USERSPACE default 10 help Each table can address up to 4MB memory address. @@ -159,6 +170,15 @@ if XTENSA_MMU endif # XTENSA_MMU +config XTENSA_SYSCALL_USE_HELPER + bool "Use userspace syscall helper" + default y if "$(ZEPHYR_TOOLCHAIN_VARIANT)" = "xcc-clang" + depends on USERSPACE + help + Use syscall helpers for passing more then 3 arguments. + This is a workaround for toolchains where they have + issue modeling register usage. + endif # CPU_HAS_MMU endmenu diff --git a/arch/xtensa/core/CMakeLists.txt b/arch/xtensa/core/CMakeLists.txt index c6fffd4f5e1..8a23b65b9a9 100644 --- a/arch/xtensa/core/CMakeLists.txt +++ b/arch/xtensa/core/CMakeLists.txt @@ -22,6 +22,8 @@ zephyr_library_sources_ifdef(CONFIG_DEBUG_COREDUMP coredump.c) zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c) zephyr_library_sources_ifdef(CONFIG_GDBSTUB gdbstub.c) zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU xtensa_mmu.c) +zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace.S) +zephyr_library_sources_ifdef(CONFIG_XTENSA_SYSCALL_USE_HELPER syscall_helper.c) zephyr_library_sources_ifdef( CONFIG_KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index e693937f99b..3117ebc4a56 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -15,6 +15,7 @@ #endif #endif #include +#include #include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); @@ -120,6 +121,14 @@ void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf) z_fatal_error(reason, esf); } +#ifdef CONFIG_USERSPACE +Z_EXC_DECLARE(z_xtensa_user_string_nlen); + +static const struct z_exc_handle exceptions[] = { + Z_EXC_HANDLE(z_xtensa_user_string_nlen) +}; +#endif /* CONFIG_USERSPACE */ + #ifdef XT_SIMULATOR void exit(int return_code) { diff --git a/arch/xtensa/core/include/xtensa_mmu_priv.h b/arch/xtensa/core/include/xtensa_mmu_priv.h index dcfb9deefc0..cf72c921383 100644 --- a/arch/xtensa/core/include/xtensa_mmu_priv.h +++ b/arch/xtensa/core/include/xtensa_mmu_priv.h @@ -18,18 +18,37 @@ #define Z_XTENSA_PTE_VPN_MASK 0xFFFFF000U #define Z_XTENSA_PTE_PPN_MASK 0xFFFFF000U #define Z_XTENSA_PTE_ATTR_MASK 0x0000000FU +#define Z_XTENSA_PTE_ATTR_CACHED_MASK 0x0000000CU #define Z_XTENSA_L1_MASK 0x3FF00000U #define Z_XTENSA_L2_MASK 0x3FFFFFU #define Z_XTENSA_PPN_SHIFT 12U #define Z_XTENSA_PTE_RING_MASK 0x00000030U +#define Z_XTENSA_PTE_RING_SHIFT 4U #define Z_XTENSA_PTE(paddr, ring, attr) \ (((paddr) & Z_XTENSA_PTE_PPN_MASK) | \ - (((ring) << 4) & Z_XTENSA_PTE_RING_MASK) | \ + (((ring) << Z_XTENSA_PTE_RING_SHIFT) & Z_XTENSA_PTE_RING_MASK) | \ ((attr) & Z_XTENSA_PTE_ATTR_MASK)) +#define Z_XTENSA_PTE_ATTR_GET(pte) \ + (pte) & Z_XTENSA_PTE_ATTR_MASK + +#define Z_XTENSA_PTE_ATTR_SET(pte, attr) \ + (((pte) & ~Z_XTENSA_PTE_ATTR_MASK) | (attr)) + +#define Z_XTENSA_PTE_RING_SET(pte, ring) \ + (((pte) & ~Z_XTENSA_PTE_RING_MASK) | \ + ((ring) << Z_XTENSA_PTE_RING_SHIFT)) + +#define Z_XTENSA_PTE_RING_GET(pte) \ + (((pte) & ~Z_XTENSA_PTE_RING_MASK) >> Z_XTENSA_PTE_RING_SHIFT) + +#define Z_XTENSA_PTE_ASID_GET(pte, rasid) \ + (((rasid) >> ((((pte) & Z_XTENSA_PTE_RING_MASK) \ + >> Z_XTENSA_PTE_RING_SHIFT) * 8)) & 0xFF) + #define Z_XTENSA_TLB_ENTRY(vaddr, way) \ (((vaddr) & Z_XTENSA_PTE_PPN_MASK) | (way)) @@ -38,11 +57,38 @@ (((vaddr) >> Z_XTENSA_PPN_SHIFT) & 0x03U)) #define Z_XTENSA_L2_POS(vaddr) \ - (((vaddr) & Z_XTENSA_L2_MASK) >> Z_XTENSA_PPN_SHIFT) + (((vaddr) & Z_XTENSA_L2_MASK) >> 12U) + +#define Z_XTENSA_L1_POS(vaddr) \ + ((vaddr) >> 22U) + +/* PTE attributes for entries in the L1 page table. Should never be + * writable, may be cached in non-SMP contexts only + */ +#if CONFIG_MP_MAX_NUM_CPUS == 1 +#define Z_XTENSA_PAGE_TABLE_ATTR Z_XTENSA_MMU_CACHED_WB +#else +#define Z_XTENSA_PAGE_TABLE_ATTR 0 +#endif + +/* This ASID is shared between all domains and kernel. */ +#define Z_XTENSA_MMU_SHARED_ASID 255 + +/* Fixed data TLB way to map the page table */ +#define Z_XTENSA_MMU_PTE_WAY 7 + +/* Fixed data TLB way to map the vecbase */ +#define Z_XTENSA_MMU_VECBASE_WAY 8 /* Kernel specific ASID. Ring field in the PTE */ #define Z_XTENSA_KERNEL_RING 0 +/* User specific ASID. Ring field in the PTE */ +#define Z_XTENSA_USER_RING 2 + +/* Ring value for MMU_SHARED_ASID */ +#define Z_XTENSA_SHARED_RING 3 + /* Number of data TLB ways [0-9] */ #define Z_XTENSA_DTLB_WAYS 10 @@ -96,6 +142,14 @@ #define Z_XTENSA_PAGE_TABLE_VADDR \ Z_XTENSA_PTE_ENTRY_VADDR(Z_XTENSA_PTEVADDR) +/* + * Get asid for a given ring from rasid register. + * rasid contains four asid, one per ring. + */ + +#define Z_XTENSA_RASID_ASID_GET(rasid, ring) \ + (((rasid) >> ((ring) * 8)) & 0xff) + static ALWAYS_INLINE void xtensa_rasid_set(uint32_t rasid) { __asm__ volatile("wsr %0, rasid\n\t" @@ -110,6 +164,16 @@ static ALWAYS_INLINE uint32_t xtensa_rasid_get(void) return rasid; } +static ALWAYS_INLINE void xtensa_rasid_asid_set(uint8_t asid, uint8_t pos) +{ + uint32_t rasid = xtensa_rasid_get(); + + rasid = (rasid & ~(0xff << (pos * 8))) | ((uint32_t)asid << (pos * 8)); + + xtensa_rasid_set(rasid); +} + + static ALWAYS_INLINE void xtensa_itlb_entry_invalidate(uint32_t entry) { __asm__ volatile("iitlb %0\n\t" @@ -201,6 +265,21 @@ static ALWAYS_INLINE void xtensa_ptevaddr_set(void *ptables) __asm__ volatile("wsr.ptevaddr %0" : : "a"((uint32_t)ptables)); } +/** + * @brief Get the current page tables. + * + * The page tables is obtained by reading ptevaddr address. + * + * @return ptables The page tables address (virtual address) + */ +static ALWAYS_INLINE void *xtensa_ptevaddr_get(void) +{ + uint32_t ptables; + + __asm__ volatile("rsr.ptevaddr %0" : "=a" (ptables)); + + return (void *)ptables; +} /* * The following functions are helpful when debugging. */ diff --git a/arch/xtensa/core/offsets/offsets.c b/arch/xtensa/core/offsets/offsets.c index 860a12eb408..8d4022fd81e 100644 --- a/arch/xtensa/core/offsets/offsets.c +++ b/arch/xtensa/core/offsets/offsets.c @@ -5,6 +5,7 @@ #include #include +#include #include @@ -60,4 +61,10 @@ GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu14); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu15); #endif +#ifdef CONFIG_USERSPACE +GEN_OFFSET_SYM(_thread_arch_t, psp); +GEN_OFFSET_SYM(_thread_arch_t, ptables); +#endif + + GEN_ABS_SYM_END diff --git a/arch/xtensa/core/syscall_helper.c b/arch/xtensa/core/syscall_helper.c new file mode 100644 index 00000000000..fbbdf104191 --- /dev/null +++ b/arch/xtensa/core/syscall_helper.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + register uintptr_t a9 __asm__("%a9") = arg6; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8), "r" (a9) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2) + : "memory"); + + return a2; +} diff --git a/arch/xtensa/core/userspace.S b/arch/xtensa/core/userspace.S new file mode 100644 index 00000000000..b649014fd52 --- /dev/null +++ b/arch/xtensa/core/userspace.S @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2022, Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +/** + * syscall number arg1, arg2, arg3, arg4, arg5, arg6 + * -------------- ---------------------------------- + * a2 a6, a3, a4, a5, a8, a9 + * + **/ +.pushsection .text.z_xtensa_do_syscall, "ax" +.global z_xtensa_do_syscall +.align 4 +z_xtensa_do_syscall: + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + l32i a0, a0, _thread_offset_to_psp + + addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF + + s32i a1, a0, ___xtensa_irq_bsa_t_scratch_OFFSET + s32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET + s32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET + rsr a2, ZSR_A0SAVE + s32i a2, a0, ___xtensa_irq_bsa_t_a0_OFFSET + rsr.ps a2 + movi a3, ~PS_OWB_MASK + and a2, a2, a3 + s32i a2, a0, ___xtensa_irq_bsa_t_ps_OFFSET + rsr.epc1 a2 + s32i a2, a0, ___xtensa_irq_bsa_t_pc_OFFSET + + movi a2, PS_WOE|PS_INTLEVEL(XCHAL_NMILEVEL) + rsr.ps a3 + or a3, a3, a2 + movi a2, ~(PS_EXCM | PS_RING_MASK) + and a3, a3, a2 + wsr.ps a3 + rsync + l32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET + l32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET + SPILL_ALL_WINDOWS + + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + l32i a0, a0, _thread_offset_to_psp + addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF + + mov a1, a0 + + l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET +#if XCHAL_HAVE_LOOPS + /* If the syscall instruction was the last instruction in the body of + * a zero-overhead loop, and the loop will execute again, decrement + * the loop count and resume execution at the head of the loop. + */ + rsr.lend a2 + addi a3, a3, 3 + bne a2, a3, end_loop + rsr.lcount a2 + beqz a2, end_loop + addi a2, a2, -1 + wsr.lcount a2 + rsr.lbeg a3 +end_loop: +#else + /* EPC1 (and now a3) contains the address that invoked syscall. + * We need to increment it to execute the next instruction when + * we return. The instruction size is 3 bytes, so lets just add it. + */ + addi a3, a3, 3 +#endif + s32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET + ODD_REG_SAVE + + call0 xtensa_save_high_regs + + l32i a2, a1, 0 + l32i a2, a2, ___xtensa_irq_bsa_t_a2_OFFSET + movi a0, K_SYSCALL_LIMIT + bgeu a2, a0, _bad_syscall + +_id_ok: + /* Find the function handler for the given syscall id. */ + movi a3, _k_syscall_table + slli a2, a2, 2 + add a2, a2, a3 + l32i a2, a2, 0 + + /* Clear up the threadptr because it is used + * to check if a thread is running on user mode. Since + * we are in a interruption we don't want the system + * thinking it is possibly running in user mode. + */ + movi a0, 0 + wur.THREADPTR a0 + + /* Set syscall parameters. We have an initial call4 to set up the + * the stack and then a new call4 for the syscall function itself. + * So parameters should be put as if it was a call8. + */ + mov a10, a8 + mov a11, a9 + mov a8, a4 + mov a9, a5 + l32i a3, a1, 0 + l32i a7, a3, ___xtensa_irq_bsa_t_a3_OFFSET + + + /* Since we are unmasking EXCM, we need to set RING bits to kernel + * mode, otherwise we won't be able to run the exception handler in C. + */ + movi a0, PS_WOE|PS_CALLINC(0)|PS_UM|PS_INTLEVEL(0) + wsr.ps a0 + rsync + + call4 _syscall_call0 + + /* copy return value. Lets put it in the top of stack + * because registers will be clobbered in + * xtensa_restore_high_regs + */ + l32i a3, a1, 0 + s32i a6, a3, ___xtensa_irq_bsa_t_a2_OFFSET + + j _syscall_returned + +.align 4 +_syscall_call0: + /* We want an ENTRY to set a bit in windowstart */ + jx a2 + + +_syscall_returned: + call0 xtensa_restore_high_regs + + l32i a3, a1, ___xtensa_irq_bsa_t_sar_OFFSET + wsr a3, SAR +#if XCHAL_HAVE_LOOPS + l32i a3, a1, ___xtensa_irq_bsa_t_lbeg_OFFSET + wsr a3, LBEG + l32i a3, a1, ___xtensa_irq_bsa_t_lend_OFFSET + wsr a3, LEND + l32i a3, a1, ___xtensa_irq_bsa_t_lcount_OFFSET + wsr a3, LCOUNT +#endif +#if XCHAL_HAVE_S32C1I + l32i a3, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET + wsr a3, SCOMPARE1 +#endif + + rsr a3, ZSR_CPU + l32i a3, a3, ___cpu_t_current_OFFSET + wur.THREADPTR a3 + + l32i a3, a1, ___xtensa_irq_bsa_t_ps_OFFSET + wsr.ps a3 + + l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET + wsr.epc1 a3 + + l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET + l32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET + l32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET + + l32i a1, a1, ___xtensa_irq_bsa_t_scratch_OFFSET + rsync + + rfe + +_bad_syscall: + movi a2, K_SYSCALL_BAD + j _id_ok + +.popsection + +/* FUNC_NORETURN void z_xtensa_userspace_enter(k_thread_entry_t user_entry, + * void *p1, void *p2, void *p3, + * uint32_t stack_end, + * uint32_t stack_start) + * + * A one-way trip to userspace. + */ +.global z_xtensa_userspace_enter +.type z_xtensa_userspace_enter, @function +.align 4 +z_xtensa_userspace_enter: + /* Call entry to set a bit in the windowstart and + * do the rotation, but we are going to set our own + * stack. + */ + entry a1, 16 + + /* We have to switch to kernel stack before spill kernel data and + * erase user stack to avoid leak from previous context. + */ + mov a1, a7 /* stack start (low address) */ + addi a1, a1, -16 + + SPILL_ALL_WINDOWS + + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + + addi a1, a1, -28 + s32i a0, a1, 24 + s32i a2, a1, 20 + s32i a3, a1, 16 + s32i a4, a1, 12 + s32i a5, a1, 8 + s32i a6, a1, 4 + s32i a7, a1, 0 + + l32i a6, a1, 24 + call4 xtensa_user_stack_perms + + l32i a6, a1, 24 + call4 z_xtensa_swap_update_page_tables + + /* Set threadptr with the thread address, we are going to user mode. */ + l32i a0, a1, 24 + wur.THREADPTR a0 + + /* Set now z_thread_entry parameters, we are simulating a call4 + * call, so parameters start at a6, a7, ... + */ + l32i a6, a1, 20 + l32i a7, a1, 16 + l32i a8, a1, 12 + l32i a9, a1, 8 + + /* stash user stack */ + l32i a0, a1, 4 + + addi a1, a1, 28 + + /* Go back to user stack */ + mov a1, a0 + + movi a0, z_thread_entry + wsr.epc2 a0 + + /* Configuring PS register. + * We have to set callinc as well, since the called + * function will do "entry" + */ + movi a0, PS_WOE|PS_CALLINC(1)|PS_UM|PS_RING(2) + wsr a0, EPS2 + + movi a0, 0 + + rfi 2 + +/* + * size_t arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg) + */ +.global arch_user_string_nlen +.type arch_user_string_nlen, @function +.align 4 +arch_user_string_nlen: + entry a1, 32 + + /* error value, set to -1. */ + movi a5, -1 + s32i a5, a4, 0 + + /* length count */ + xor a5, a5, a5 + + /* This code might page fault */ +strlen_loop: +.global z_xtensa_user_string_nlen_fault_start +z_xtensa_user_string_nlen_fault_start: + l8ui a6, a2, 0 /* Current char */ + +.global z_xtensa_user_string_nlen_fault_end +z_xtensa_user_string_nlen_fault_end: + beqz a6, strlen_done + addi a5, a5, 1 + addi a2, a2, 1 + beq a5, a3, strlen_done + j strlen_loop + +strlen_done: + /* Set return value */ + mov a2, a5 + + /* Set error value to 0 since we succeeded */ + movi a5, 0x0 + s32i a5, a4, 0 + +.global z_xtensa_user_string_nlen_fixup +z_xtensa_user_string_nlen_fixup: + retw diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index bc526ae0165..c108a09dee4 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -174,7 +174,8 @@ _restore_context: l32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET wsr a0, SCOMPARE1 #endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR && \ + (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) l32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET wur a0, THREADPTR #endif @@ -258,6 +259,16 @@ noflush: l32i a3, a2, ___xtensa_irq_bsa_t_a3_OFFSET s32i a1, a3, 0 +#ifdef CONFIG_USERSPACE + /* Switch page tables */ + rsr a6, ZSR_CPU + l32i a6, a6, ___cpu_t_current_OFFSET + call4 z_xtensa_swap_update_page_tables + + l32i a2, a3, 0 + l32i a2, a2, 0 +#endif + /* Switch stack pointer and restore. The jump to * _restore_context does not return as such, but we arrange * for the restored "next" address to be immediately after for @@ -347,6 +358,9 @@ _Level1RealVector: rsr.exccause a0 #ifdef CONFIG_XTENSA_MMU beqi a0, EXCCAUSE_ITLB_MISS, _handle_tlb_miss_user +#ifdef CONFIG_USERSPACE + beqi a0, EXCCAUSE_SYSCALL, _syscall +#endif /* CONFIG_USERSPACE */ addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_user rsr.exccause a0 @@ -376,6 +390,11 @@ _handle_tlb_miss_user: l32i a0, a0, 0 rsr a0, ZSR_A0SAVE rfe +#ifdef CONFIG_USERSPACE +_syscall: + rsr a0, ZSR_A0SAVE + j z_xtensa_do_syscall +#endif /* CONFIG_USERSPACE */ #endif /* CONFIG_XTENSA_MMU */ .popsection diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index c2371ac8f55..1e8c92759e4 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -26,6 +26,13 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, { void *ret; _xtensa_irq_stack_frame_a11_t *frame; +#ifdef CONFIG_USERSPACE + struct z_xtensa_thread_stack_header *header = + (struct z_xtensa_thread_stack_header *)thread->stack_obj; + + thread->arch.psp = header->privilege_stack + + sizeof(header->privilege_stack); +#endif /* Not-a-cpu ID Ensures that the first time this is run, the * stack will be invalidated. That covers the edge case of @@ -48,11 +55,23 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, (void)memset(frame, 0, bsasz); - frame->bsa.pc = (uintptr_t)z_thread_entry; frame->bsa.ps = PS_WOE | PS_UM | PS_CALLINC(1); +#ifdef CONFIG_USERSPACE + if ((thread->base.user_options & K_USER) == K_USER) { + frame->bsa.pc = (uintptr_t)arch_user_mode_enter; + } else { + frame->bsa.pc = (uintptr_t)z_thread_entry; + } +#else + frame->bsa.pc = (uintptr_t)z_thread_entry; +#endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR +#ifdef CONFIG_THREAD_LOCAL_STORAGE frame->bsa.threadptr = thread->tls; +#elif CONFIG_USERSPACE + frame->bsa.threadptr = (uintptr_t)((thread->base.user_options & K_USER) ? thread : NULL); +#endif #endif /* Arguments to z_thread_entry(). Remember these start at A6, @@ -471,3 +490,24 @@ void arch_spin_relax(void) #undef NOP1 } #endif /* CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS */ + +#ifdef CONFIG_USERSPACE +FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3) +{ + struct k_thread *current = _current; + size_t stack_end; + + /* Transition will reset stack pointer to initial, discarding + * any old context since this is a one-way operation + */ + stack_end = Z_STACK_PTR_ALIGN(current->stack_info.start + + current->stack_info.size - + current->stack_info.delta); + + z_xtensa_userspace_enter(user_entry, p1, p2, p3, + stack_end, current->stack_info.start); + + CODE_UNREACHABLE; +} +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index 18ce5f22f63..915a26357eb 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include #include @@ -15,22 +16,24 @@ #include #include -/* Fixed data TLB way to map the page table */ -#define MMU_PTE_WAY 7 - -/* Fixed data TLB way to map VECBASE */ -#define MMU_VECBASE_WAY 8 - /* Level 1 contains page table entries * necessary to map the page table itself. */ #define XTENSA_L1_PAGE_TABLE_ENTRIES 1024U +/* Size of level 1 page table. + */ +#define XTENSA_L1_PAGE_TABLE_SIZE (XTENSA_L1_PAGE_TABLE_ENTRIES * sizeof(uint32_t)) + /* Level 2 contains page table entries * necessary to map the page table itself. */ #define XTENSA_L2_PAGE_TABLE_ENTRIES 1024U +/* Size of level 2 page table. + */ +#define XTENSA_L2_PAGE_TABLE_SIZE (XTENSA_L2_PAGE_TABLE_ENTRIES * sizeof(uint32_t)) + LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, @@ -40,8 +43,18 @@ BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, * Level 1 page table has to be 4Kb to fit into one of the wired entries. * All entries are initialized as INVALID, so an attempt to read an unmapped * area will cause a double exception. + * + * Each memory domain contains its own l1 page table. The kernel l1 page table is + * located at the index 0. */ -uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4)); +static uint32_t l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES] + __aligned(KB(4)); + + +/* + * That is an alias for the page tables set used by the kernel. + */ +uint32_t *z_xtensa_kernel_ptables = (uint32_t *)l1_page_table[0]; /* * Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one @@ -50,12 +63,41 @@ uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4)); static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_TABLE_ENTRIES] __aligned(KB(4)); +/* + * This additional variable tracks which l1 tables are in use. This is kept separated from + * the tables to keep alignment easier. + * + * @note: The first bit is set because it is used for the kernel page tables. + */ +static ATOMIC_DEFINE(l1_page_table_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES); + /* * This additional variable tracks which l2 tables are in use. This is kept separated from * the tables to keep alignment easier. */ static ATOMIC_DEFINE(l2_page_tables_track, CONFIG_XTENSA_MMU_NUM_L2_TABLES); +/* + * Protects xtensa_domain_list and serializes access to page tables. + */ +static struct k_spinlock xtensa_mmu_lock; + +#ifdef CONFIG_USERSPACE + +/* + * Each domain has its own ASID. ASID can go through 1 (kernel) to 255. + * When a TLB entry matches, the hw will check the ASID in the entry and finds + * the correspondent position in the RASID register. This position will then be + * compared with the current ring (CRING) to check the permission. + */ +static uint8_t asid_count = 3; + +/* + * List with all active and initialized memory domains. + */ +static sys_slist_t xtensa_domain_list; +#endif /* CONFIG_USERSPACE */ + extern char _heap_end[]; extern char _heap_start[]; extern char __data_start[]; @@ -100,18 +142,61 @@ static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { { .start = (uint32_t)__text_region_start, .end = (uint32_t)__text_region_end, - .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB, + .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, .name = "text", }, /* Mark rodata segment cacheable, read only and non-executable */ { .start = (uint32_t)__rodata_region_start, .end = (uint32_t)__rodata_region_end, - .attrs = Z_XTENSA_MMU_CACHED_WB, + .attrs = Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, .name = "rodata", }, }; +static inline uint32_t *thread_page_tables_get(const struct k_thread *thread) +{ +#ifdef CONFIG_USERSPACE + if ((thread->base.user_options & K_USER) != 0U) { + return thread->arch.ptables; + } +#endif + + return z_xtensa_kernel_ptables; +} + +/** + * @brief Check if the page table entry is illegal. + * + * @param[in] Page table entry. + */ +static inline bool is_pte_illegal(uint32_t pte) +{ + uint32_t attr = pte & Z_XTENSA_PTE_ATTR_MASK; + + /* + * The ISA manual states only 12 and 14 are illegal values. + * 13 and 15 are not. So we need to be specific than simply + * testing if bits 2 and 3 are set. + */ + return (attr == 12) || (attr == 14); +} + +/* + * @brief Initialize all page table entries to be illegal. + * + * @param[in] Pointer to page table. + * @param[in] Number of page table entries in the page table. + */ +static void init_page_table(uint32_t *ptable, size_t num_entries) +{ + int i; + + for (i = 0; i < num_entries; i++) { + ptable[i] = Z_XTENSA_MMU_ILLEGAL; + } +} + static inline uint32_t *alloc_l2_table(void) { uint16_t idx; @@ -125,45 +210,86 @@ static inline uint32_t *alloc_l2_table(void) return NULL; } +/** + * @brief Switch page tables + * + * This switches the page tables to the incoming ones (@a ptables). + * Since data TLBs to L2 page tables are auto-filled, @a dtlb_inv + * can be used to invalidate these data TLBs. @a cache_inv can be + * set to true to invalidate cache to the page tables. + * + * @param[in] ptables Page tables to be switched to. + * @param[in] dtlb_inv True if to invalidate auto-fill data TLBs. + * @param[in] cache_inv True if to invalidate cache to page tables. + */ +static ALWAYS_INLINE void switch_page_tables(uint32_t *ptables, bool dtlb_inv, bool cache_inv) +{ + if (cache_inv) { + sys_cache_data_invd_range((void *)ptables, XTENSA_L1_PAGE_TABLE_SIZE); + sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + } + + /* Invalidate data TLB to L1 page table */ + xtensa_dtlb_vaddr_invalidate((void *)Z_XTENSA_PAGE_TABLE_VADDR); + + /* Now map the pagetable itself with KERNEL asid to avoid user thread + * from tampering with it. + */ + xtensa_dtlb_entry_write_sync( + Z_XTENSA_PTE((uint32_t)ptables, Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR), + Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY)); + + if (dtlb_inv) { + /* Since L2 page tables are auto-refilled, + * invalidate all of them to flush the old entries out. + */ + xtensa_tlb_autorefill_invalidate(); + } +} + static void map_memory_range(const uint32_t start, const uint32_t end, - const uint32_t attrs) + const uint32_t attrs, bool shared) { uint32_t page, *table; for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) { - uint32_t pte = Z_XTENSA_PTE(page, Z_XTENSA_KERNEL_RING, attrs); + uint32_t pte = Z_XTENSA_PTE(page, + shared ? Z_XTENSA_SHARED_RING : Z_XTENSA_KERNEL_RING, + attrs); uint32_t l2_pos = Z_XTENSA_L2_POS(page); - uint32_t l1_pos = page >> 22; + uint32_t l1_pos = Z_XTENSA_L1_POS(page); - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { + if (is_pte_illegal(z_xtensa_kernel_ptables[l1_pos])) { table = alloc_l2_table(); __ASSERT(table != NULL, "There is no l2 page table available to " "map 0x%08x\n", page); - l1_page_table[l1_pos] = + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + + z_xtensa_kernel_ptables[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT); + Z_XTENSA_PAGE_TABLE_ATTR); } - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + table = (uint32_t *)(z_xtensa_kernel_ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); table[l2_pos] = pte; } } static void map_memory(const uint32_t start, const uint32_t end, - const uint32_t attrs) + const uint32_t attrs, bool shared) { - map_memory_range(start, end, attrs); + map_memory_range(start, end, attrs, shared); #ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP if (arch_xtensa_is_ptr_uncached((void *)start)) { map_memory_range(POINTER_TO_UINT(z_soc_cached_ptr((void *)start)), POINTER_TO_UINT(z_soc_cached_ptr((void *)end)), - attrs | Z_XTENSA_MMU_CACHED_WB); + attrs | Z_XTENSA_MMU_CACHED_WB, shared); } else if (arch_xtensa_is_ptr_cached((void *)start)) { map_memory_range(POINTER_TO_UINT(z_soc_uncached_ptr((void *)start)), - POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs); + POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs, shared); } #endif } @@ -171,16 +297,19 @@ static void map_memory(const uint32_t start, const uint32_t end, static void xtensa_init_page_tables(void) { volatile uint8_t entry; - uint32_t page; - for (page = 0; page < XTENSA_L1_PAGE_TABLE_ENTRIES; page++) { - l1_page_table[page] = Z_XTENSA_MMU_ILLEGAL; - } + init_page_table(z_xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES); + atomic_set_bit(l1_page_table_track, 0); for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) { const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry]; + bool shared; + uint32_t attrs; - map_memory(range->start, range->end, range->attrs); + shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED); + attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED; + + map_memory(range->start, range->end, attrs, shared); } /** @@ -198,8 +327,13 @@ static void xtensa_init_page_tables(void) #endif for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) { const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry]; + bool shared; + uint32_t attrs; - map_memory(range->start, range->end, range->attrs); + shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED); + attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED; + + map_memory(range->start, range->end, attrs, shared); } #if defined(__GNUC__) #pragma GCC diagnostic pop @@ -231,6 +365,9 @@ void z_xtensa_mmu_init(void) /* Set the page table location in the virtual address */ xtensa_ptevaddr_set((void *)Z_XTENSA_PTEVADDR); + /* Set rasid */ + xtensa_rasid_asid_set(Z_XTENSA_MMU_SHARED_ASID, Z_XTENSA_SHARED_RING); + /* Next step is to invalidate the tlb entry that contains the top level * page table. This way we don't cause a multi hit exception. */ @@ -243,9 +380,9 @@ void z_xtensa_mmu_init(void) * Lets use one of the wired entry, so we never have tlb miss for * the top level table. */ - xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)l1_page_table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, MMU_PTE_WAY)); + xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)z_xtensa_kernel_ptables, + Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR), + Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY)); /* Before invalidate the text region in the TLB entry 6, we need to * map the exception vector into one of the wired entries to avoid @@ -297,7 +434,7 @@ void z_xtensa_mmu_init(void) xtensa_dtlb_entry_write( Z_XTENSA_PTE((uint32_t)vecbase, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_CACHED_WB), - Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, MMU_VECBASE_WAY)); + Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, Z_XTENSA_MMU_VECBASE_WAY)); /* * Pre-load TLB for vecbase so exception handling won't result @@ -325,6 +462,12 @@ void z_xtensa_mmu_init(void) xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); + /* + * Clear out THREADPTR as we use it to indicate + * whether we are in user mode or not. + */ + XTENSA_WUR("THREADPTR", 0); + arch_xtensa_mmu_post_init(_current_cpu->id == 0); } @@ -351,32 +494,121 @@ __weak void arch_reserved_pages_update(void) } #endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ -static bool l2_page_table_map(void *vaddr, uintptr_t phys, uint32_t flags) +static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, + uint32_t flags, bool is_user) { uint32_t l1_pos = (uint32_t)vaddr >> 22; - uint32_t pte = Z_XTENSA_PTE(phys, Z_XTENSA_KERNEL_RING, flags); uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); uint32_t *table; - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + if (is_pte_illegal(l1_table[l1_pos])) { table = alloc_l2_table(); if (table == NULL) { return false; } - l1_page_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT); + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + + l1_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + + sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); } - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = pte; + table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + table[l2_pos] = Z_XTENSA_PTE(phys, is_user ? Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING, + flags); - if ((flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) { + sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0])); + + return true; +} + +static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, bool is_user) +{ + bool ret; + void *vaddr, *vaddr_uc; + uintptr_t paddr, paddr_uc; + uint32_t flags, flags_uc; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (arch_xtensa_is_ptr_cached(va)) { + vaddr = va; + vaddr_uc = arch_xtensa_uncached_ptr(va); + } else { + vaddr = arch_xtensa_cached_ptr(va); + vaddr_uc = va; + } + + if (arch_xtensa_is_ptr_cached((void *)pa)) { + paddr = pa; + paddr_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)pa); + } else { + paddr = (uintptr_t)arch_xtensa_cached_ptr((void *)pa); + paddr_uc = pa; + } + + flags_uc = (xtensa_flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK); + flags = flags_uc | Z_XTENSA_MMU_CACHED_WB; + } else { + vaddr = va; + paddr = pa; + flags = xtensa_flags; + } + + ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr, paddr, + flags, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped", va); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { + ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc, + flags_uc, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped", vaddr_uc); + } + +#ifndef CONFIG_USERSPACE + ARG_UNUSED(ret); +#else + if (ret) { + sys_snode_t *node; + struct arch_mem_domain *domain; + k_spinlock_key_t key; + + key = k_spin_lock(&z_mem_domain_lock); + SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) { + domain = CONTAINER_OF(node, struct arch_mem_domain, node); + + ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr, + flags, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + vaddr, domain); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { + ret = l2_page_table_map(domain->ptables, + (void *)vaddr_uc, paddr_uc, + flags_uc, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + vaddr_uc, domain); + } + } + k_spin_unlock(&z_mem_domain_lock, key); + } +#endif /* CONFIG_USERSPACE */ + + if ((xtensa_flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) { xtensa_itlb_vaddr_invalidate(vaddr); } xtensa_dtlb_vaddr_invalidate(vaddr); - return true; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (xtensa_flags & Z_XTENSA_MMU_X) { + xtensa_itlb_vaddr_invalidate(vaddr_uc); + } + xtensa_dtlb_vaddr_invalidate(vaddr_uc); + } } void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) @@ -385,7 +617,8 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) uint32_t pa = (uint32_t)phys; uint32_t rem_size = (uint32_t)size; uint32_t xtensa_flags = 0; - int key; + k_spinlock_key_t key; + bool is_user; if (size == 0) { LOG_ERR("Cannot map physical memory at 0x%08X: invalid " @@ -414,63 +647,130 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) xtensa_flags |= Z_XTENSA_MMU_X; } - key = arch_irq_lock(); + is_user = (flags & K_MEM_PERM_USER) == K_MEM_PERM_USER; + + key = k_spin_lock(&xtensa_mmu_lock); while (rem_size > 0) { - bool ret = l2_page_table_map((void *)va, pa, xtensa_flags); + __arch_mem_map((void *)va, pa, xtensa_flags, is_user); - ARG_UNUSED(ret); - __ASSERT(ret, "Virtual address (%u) already mapped", (uint32_t)virt); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); pa += KB(4); } - arch_irq_unlock(key); + k_spin_unlock(&xtensa_mmu_lock, key); } -static void l2_page_table_unmap(void *vaddr) +/** + * @return True if page is executable (thus need to invalidate ITLB), + * false if not. + */ +static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) { uint32_t l1_pos = (uint32_t)vaddr >> 22; uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); - uint32_t *table; + uint32_t *l2_table; uint32_t table_pos; bool exec; - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { - return; + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + if (is_pte_illegal(l1_table[l1_pos])) { + /* We shouldn't be unmapping an illegal entry. + * Return true so that we can invalidate ITLB too. + */ + return true; } - exec = l1_page_table[l1_pos] & Z_XTENSA_MMU_X; + exec = l1_table[l1_pos] & Z_XTENSA_MMU_X; - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = Z_XTENSA_MMU_ILLEGAL; + l2_table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + + sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + l2_table[l2_pos] = Z_XTENSA_MMU_ILLEGAL; + + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) { - if (table[l2_pos] != Z_XTENSA_MMU_ILLEGAL) { + if (!is_pte_illegal(l2_table[l2_pos])) { goto end; } } - l1_page_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL; - table_pos = (table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); + l1_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL; + sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); atomic_clear_bit(l2_page_tables_track, table_pos); /* Need to invalidate L2 page table as it is no longer valid. */ - xtensa_dtlb_vaddr_invalidate((void *)table); + xtensa_dtlb_vaddr_invalidate((void *)l2_table); end: - if (exec) { + return exec; +} + +static inline void __arch_mem_unmap(void *va) +{ + bool is_exec; + void *vaddr, *vaddr_uc; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (arch_xtensa_is_ptr_cached(va)) { + vaddr = va; + vaddr_uc = arch_xtensa_uncached_ptr(va); + } else { + vaddr = arch_xtensa_cached_ptr(va); + vaddr_uc = va; + } + } else { + vaddr = va; + } + + is_exec = l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + (void)l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr_uc); + } + +#ifdef CONFIG_USERSPACE + sys_snode_t *node; + struct arch_mem_domain *domain; + k_spinlock_key_t key; + + key = k_spin_lock(&z_mem_domain_lock); + SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) { + domain = CONTAINER_OF(node, struct arch_mem_domain, node); + + (void)l2_page_table_unmap(domain->ptables, (void *)vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + (void)l2_page_table_unmap(domain->ptables, (void *)vaddr_uc); + } + } + k_spin_unlock(&z_mem_domain_lock, key); +#endif /* CONFIG_USERSPACE */ + + if (is_exec) { xtensa_itlb_vaddr_invalidate(vaddr); } xtensa_dtlb_vaddr_invalidate(vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (is_exec) { + xtensa_itlb_vaddr_invalidate(vaddr_uc); + } + xtensa_dtlb_vaddr_invalidate(vaddr_uc); + } } void arch_mem_unmap(void *addr, size_t size) { uint32_t va = (uint32_t)addr; uint32_t rem_size = (uint32_t)size; - int key; + k_spinlock_key_t key; if (addr == NULL) { LOG_ERR("Cannot unmap NULL pointer"); @@ -482,13 +782,363 @@ void arch_mem_unmap(void *addr, size_t size) return; } - key = arch_irq_lock(); + key = k_spin_lock(&xtensa_mmu_lock); while (rem_size > 0) { - l2_page_table_unmap((void *)va); + __arch_mem_unmap((void *)va); + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); } - arch_irq_unlock(key); + k_spin_unlock(&xtensa_mmu_lock, key); } + +#ifdef CONFIG_USERSPACE + +static inline uint32_t *alloc_l1_table(void) +{ + uint16_t idx; + + for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L1_TABLES; idx++) { + if (!atomic_test_and_set_bit(l1_page_table_track, idx)) { + return (uint32_t *)&l1_page_table[idx]; + } + } + + return NULL; +} + +static uint32_t *dup_table(uint32_t *source_table) +{ + uint16_t i, j; + uint32_t *dst_table = alloc_l1_table(); + + if (!dst_table) { + return NULL; + } + + for (i = 0; i < XTENSA_L1_PAGE_TABLE_ENTRIES; i++) { + uint32_t *l2_table, *src_l2_table; + + if (is_pte_illegal(source_table[i])) { + dst_table[i] = Z_XTENSA_MMU_ILLEGAL; + continue; + } + + src_l2_table = (uint32_t *)(source_table[i] & Z_XTENSA_PTE_PPN_MASK); + l2_table = alloc_l2_table(); + if (l2_table == NULL) { + goto err; + } + + for (j = 0; j < XTENSA_L2_PAGE_TABLE_ENTRIES; j++) { + l2_table[j] = src_l2_table[j]; + } + + /* The page table is using kernel ASID because we don't + * user thread manipulate it. + */ + dst_table[i] = Z_XTENSA_PTE((uint32_t)l2_table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + + sys_cache_data_flush_range((void *)l2_table, XTENSA_L2_PAGE_TABLE_SIZE); + } + + sys_cache_data_flush_range((void *)dst_table, XTENSA_L1_PAGE_TABLE_SIZE); + + return dst_table; + +err: + /* TODO: Cleanup failed allocation*/ + return NULL; +} + +int arch_mem_domain_init(struct k_mem_domain *domain) +{ + uint32_t *ptables; + k_spinlock_key_t key; + int ret; + + /* + * For now, lets just assert if we have reached the maximum number + * of asid we assert. + */ + __ASSERT(asid_count < (Z_XTENSA_MMU_SHARED_ASID), "Reached maximum of ASID available"); + + key = k_spin_lock(&xtensa_mmu_lock); + ptables = dup_table(z_xtensa_kernel_ptables); + + if (ptables == NULL) { + ret = -ENOMEM; + goto err; + } + + domain->arch.ptables = ptables; + domain->arch.asid = ++asid_count; + + sys_slist_append(&xtensa_domain_list, &domain->arch.node); + + ret = 0; + +err: + k_spin_unlock(&xtensa_mmu_lock, key); + + return ret; +} + +static int region_map_update(uint32_t *ptables, uintptr_t start, + size_t size, uint32_t ring, uint32_t flags) +{ + int ret = 0; + + for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { + uint32_t *l2_table, pte; + uint32_t page = start + offset; + uint32_t l1_pos = page >> 22; + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + + /* Make sure we grab a fresh copy of L1 page table */ + sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0])); + + l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + + sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + pte = Z_XTENSA_PTE_RING_SET(l2_table[l2_pos], ring); + pte = Z_XTENSA_PTE_ATTR_SET(pte, flags); + + l2_table[l2_pos] = pte; + + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + xtensa_dtlb_vaddr_invalidate( + (void *)(pte & Z_XTENSA_PTE_PPN_MASK)); + } + + return ret; +} + +static inline int update_region(uint32_t *ptables, uintptr_t start, + size_t size, uint32_t ring, uint32_t flags) +{ + int ret; + k_spinlock_key_t key; + + key = k_spin_lock(&xtensa_mmu_lock); + +#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP + uintptr_t va, va_uc; + uint32_t new_flags, new_flags_uc; + + if (arch_xtensa_is_ptr_cached((void *)start)) { + va = start; + va_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)start); + } else { + va = (uintptr_t)arch_xtensa_cached_ptr((void *)start); + va_uc = start; + } + + new_flags_uc = (flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK); + new_flags = new_flags_uc | Z_XTENSA_MMU_CACHED_WB; + + ret = region_map_update(ptables, va, size, ring, new_flags); + + if (ret == 0) { + ret = region_map_update(ptables, va_uc, size, ring, new_flags_uc); + } +#else + ret = region_map_update(ptables, start, size, ring, flags); +#endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ + + k_spin_unlock(&xtensa_mmu_lock, key); + + return ret; +} + +static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size) +{ + return update_region(ptables, start, size, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_W); +} + +void xtensa_set_stack_perms(struct k_thread *thread) +{ + if ((thread->base.user_options & K_USER) == 0) { + return; + } + + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB); +} + +void xtensa_user_stack_perms(struct k_thread *thread) +{ + (void)memset((void *)thread->stack_info.start, 0xAA, + thread->stack_info.size - thread->stack_info.delta); + + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB); +} + +int arch_mem_domain_max_partitions_get(void) +{ + return CONFIG_MAX_DOMAIN_PARTITIONS; +} + +int arch_mem_domain_partition_remove(struct k_mem_domain *domain, + uint32_t partition_id) +{ + struct k_mem_partition *partition = &domain->partitions[partition_id]; + + /* Reset the partition's region back to defaults */ + return reset_region(domain->arch.ptables, partition->start, + partition->size); +} + +int arch_mem_domain_partition_add(struct k_mem_domain *domain, + uint32_t partition_id) +{ + uint32_t ring = domain->arch.asid == 0 ? Z_XTENSA_KERNEL_RING : Z_XTENSA_USER_RING; + struct k_mem_partition *partition = &domain->partitions[partition_id]; + + return update_region(domain->arch.ptables, partition->start, + partition->size, ring, partition->attr); +} + +/* These APIs don't need to do anything */ +int arch_mem_domain_thread_add(struct k_thread *thread) +{ + int ret = 0; + bool is_user, is_migration; + uint32_t *old_ptables; + struct k_mem_domain *domain; + + old_ptables = thread->arch.ptables; + domain = thread->mem_domain_info.mem_domain; + thread->arch.ptables = domain->arch.ptables; + + is_user = (thread->base.user_options & K_USER) != 0; + is_migration = (old_ptables != NULL) && is_user; + + /* Give access to the thread's stack in its new + * memory domain if it is migrating. + */ + if (is_migration) { + xtensa_set_stack_perms(thread); + } + + if (is_migration) { + ret = reset_region(old_ptables, + thread->stack_info.start, + thread->stack_info.size); + } + + return ret; +} + +int arch_mem_domain_thread_remove(struct k_thread *thread) +{ + struct k_mem_domain *domain = thread->mem_domain_info.mem_domain; + + if ((thread->base.user_options & K_USER) == 0) { + return 0; + } + + if ((thread->base.thread_state & _THREAD_DEAD) == 0) { + /* Thread is migrating to another memory domain and not + * exiting for good; we weren't called from + * z_thread_abort(). Resetting the stack region will + * take place in the forthcoming thread_add() call. + */ + return 0; + } + + /* Restore permissions on the thread's stack area since it is no + * longer a member of the domain. + */ + return reset_region(domain->arch.ptables, + thread->stack_info.start, + thread->stack_info.size); +} + +static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool write) +{ + uint8_t asid_ring; + uint32_t rasid, pte, *l2_table; + uint32_t l1_pos = page >> 22; + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + + if (is_pte_illegal(ptables[l1_pos])) { + return false; + } + + l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + pte = l2_table[l2_pos]; + + if (is_pte_illegal(pte)) { + return false; + } + + asid_ring = 0; + rasid = xtensa_rasid_get(); + for (uint32_t i = 0; i < 4; i++) { + if (Z_XTENSA_PTE_ASID_GET(pte, rasid) == + Z_XTENSA_RASID_ASID_GET(rasid, i)) { + asid_ring = i; + break; + } + } + + if (ring > asid_ring) { + return false; + } + + if (write) { + return (Z_XTENSA_PTE_ATTR_GET((pte)) & Z_XTENSA_MMU_W) != 0; + } + + return true; +} + +int arch_buffer_validate(void *addr, size_t size, int write) +{ + int ret = 0; + uint8_t *virt; + size_t aligned_size; + const struct k_thread *thread = _current; + uint32_t *ptables = thread_page_tables_get(thread); + uint8_t ring = ((thread->base.user_options & K_USER) != 0) ? + Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING; + + /* addr/size arbitrary, fix this up into an aligned region */ + k_mem_region_align((uintptr_t *)&virt, &aligned_size, + (uintptr_t)addr, size, CONFIG_MMU_PAGE_SIZE); + + for (size_t offset = 0; offset < aligned_size; + offset += CONFIG_MMU_PAGE_SIZE) { + if (!page_validate(ptables, (uint32_t)(virt + offset), ring, write)) { + ret = -1; + break; + } + } + + return ret; +} + +void z_xtensa_swap_update_page_tables(struct k_thread *incoming) +{ + uint32_t *ptables = incoming->arch.ptables; + struct arch_mem_domain *domain = + &(incoming->mem_domain_info.mem_domain->arch); + + /* Lets set the asid for the incoming thread */ + if ((incoming->base.user_options & K_USER) != 0) { + xtensa_rasid_asid_set(domain->asid, Z_XTENSA_USER_RING); + } + + switch_page_tables(ptables, true, false); +} + +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/include/kernel_arch_func.h b/arch/xtensa/include/kernel_arch_func.h index 2256f72f645..8ce5cc52a5b 100644 --- a/arch/xtensa/include/kernel_arch_func.h +++ b/arch/xtensa/include/kernel_arch_func.h @@ -174,6 +174,13 @@ static inline bool arch_is_in_isr(void) return arch_curr_cpu()->nested != 0U; } +#ifdef CONFIG_USERSPACE +extern void z_xtensa_userspace_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3, + uintptr_t stack_end, + uintptr_t stack_start); +#endif /* CONFIG_USERSPACE */ + #ifdef __cplusplus } #endif diff --git a/arch/xtensa/include/offsets_short_arch.h b/arch/xtensa/include/offsets_short_arch.h index 34a4a5842cf..f19750dc0ac 100644 --- a/arch/xtensa/include/offsets_short_arch.h +++ b/arch/xtensa/include/offsets_short_arch.h @@ -2,4 +2,18 @@ * Copyright (c) 2021 Intel Corporation * SPDX-License-Identifier: Apache-2.0 */ -/* Empty File */ +#ifndef ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ +#define ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ + +#define _thread_offset_to_flags \ + (___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET) + +#ifdef CONFIG_USERSPACE +#define _thread_offset_to_psp \ + (___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET) + +#define _thread_offset_to_ptables \ + (___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET) +#endif /* CONFIG_USERSPACE */ + +#endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ */ diff --git a/arch/xtensa/include/xtensa-asm2-s.h b/arch/xtensa/include/xtensa-asm2-s.h index f691dbc6cad..a98c61db253 100644 --- a/arch/xtensa/include/xtensa-asm2-s.h +++ b/arch/xtensa/include/xtensa-asm2-s.h @@ -176,7 +176,8 @@ rsr.SCOMPARE1 a0 s32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET #endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR && \ + (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) rur.THREADPTR a0 s32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET #endif @@ -409,6 +410,16 @@ _xstack_returned_\@: l32i a2, a1, 0 l32i a2, a2, ___xtensa_irq_bsa_t_scratch_OFFSET +#if XCHAL_HAVE_THREADPTR && defined(CONFIG_USERSPACE) + /* Clear up the threadptr because it is used + * to check if a thread is runnig on user mode. Since + * we are in a interruption we don't want the system + * thinking it is possbly running in user mode. + */ + movi.n a0, 0 + wur.THREADPTR a0 +#endif /* XCHAL_HAVE_THREADPTR && CONFIG_USERSPACE */ + /* There's a gotcha with level 1 handlers: the INTLEVEL field * gets left at zero and not set like high priority interrupts * do. That works fine for exceptions, but for L1 interrupts, diff --git a/include/zephyr/arch/syscall.h b/include/zephyr/arch/syscall.h index b657717e3d4..5b41561b681 100644 --- a/include/zephyr/arch/syscall.h +++ b/include/zephyr/arch/syscall.h @@ -23,6 +23,8 @@ #include #elif defined(CONFIG_RISCV) #include +#elif defined(CONFIG_XTENSA) +#include #endif #endif /* ZEPHYR_INCLUDE_ARCH_SYSCALL_H_ */ diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 0838e68ee5a..397e5c7fa78 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -47,6 +48,15 @@ extern "C" { #endif +struct arch_mem_domain { +#ifdef CONFIG_XTENSA_MMU + uint32_t *ptables __aligned(CONFIG_MMU_PAGE_SIZE); + uint8_t asid; + bool dirty; +#endif + sys_snode_t node; +}; + extern void xtensa_arch_except(int reason_p); #define ARCH_EXCEPT(reason_p) do { \ diff --git a/include/zephyr/arch/xtensa/syscall.h b/include/zephyr/arch/xtensa/syscall.h new file mode 100644 index 00000000000..3d78827b77c --- /dev/null +++ b/include/zephyr/arch/xtensa/syscall.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2022 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @file + * @brief Xtensa specific syscall header + * + * This header contains the Xtensa specific syscall interface. It is + * included by the syscall interface architecture-abstraction header + * (include/arch/syscall.h) + */ + +#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ +#define ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ + +#ifdef CONFIG_USERSPACE +#ifndef _ASMLANGUAGE + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER +uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id); + +uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id); + +uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id); +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ + +/** + * We are following Linux Xtensa syscall ABI: + * + * syscall number arg1, arg2, arg3, arg4, arg5, arg6 + * -------------- ---------------------------------- + * a2 a6, a3, a4, a5, a8, a9 + * + **/ + +static inline uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke6_helper(arg1, arg2, arg3, + arg4, arg5, arg6, + call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + register uintptr_t a9 __asm__("%a9") = arg6; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8), "r" (a9) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke5_helper(arg1, arg2, arg3, + arg4, arg5, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke4_helper(arg1, arg2, arg3, arg4, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke3_helper(arg1, arg2, arg3, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke2_helper(arg1, arg2, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3) + : "memory"); + + return a2; +#endif +} + +static inline uintptr_t arch_syscall_invoke1(uintptr_t arg1, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke1_helper(arg1, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6) + : "memory"); + + return a2; +#endif +} + +static inline uintptr_t arch_syscall_invoke0(uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke0_helper(call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2) + : "memory"); + + return a2; +#endif +} + +/* + * There is no easy (or generic) way to figure out if a thread is runnining + * in un-privileged mode. Reading the currrent ring (PS.CRING) is a privileged + * instruction and not thread local storage is not available in xcc. + */ +static inline bool arch_is_user_context(void) +{ + uint32_t thread; + + __asm__ volatile( + "rur.THREADPTR %0\n\t" + : "=a" (thread) + ); + + return !!thread; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _ASMLANGUAGE */ +#endif /* CONFIG_USERSPACE */ +#endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ */ diff --git a/include/zephyr/arch/xtensa/thread.h b/include/zephyr/arch/xtensa/thread.h index 4ec5da1ea2c..2bebe2722bc 100644 --- a/include/zephyr/arch/xtensa/thread.h +++ b/include/zephyr/arch/xtensa/thread.h @@ -7,6 +7,7 @@ #ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_ #define ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_ +#include #ifndef _ASMLANGUAGE /* Xtensa doesn't use these structs, but Zephyr core requires they be @@ -22,6 +23,14 @@ typedef struct _callee_saved _callee_saved_t; struct _thread_arch { uint32_t last_cpu; +#ifdef CONFIG_USERSPACE + uint32_t *ptables; + + /* Initial privilege mode stack pointer when doing a system call. + * Un-set for surpervisor threads. + */ + uint8_t *psp; +#endif }; typedef struct _thread_arch _thread_arch_t; diff --git a/include/zephyr/arch/xtensa/xtensa_mmu.h b/include/zephyr/arch/xtensa/xtensa_mmu.h index 2ee3dc6c9ab..40fd0ff8f5c 100644 --- a/include/zephyr/arch/xtensa/xtensa_mmu.h +++ b/include/zephyr/arch/xtensa/xtensa_mmu.h @@ -9,8 +9,40 @@ #define Z_XTENSA_MMU_X BIT(0) #define Z_XTENSA_MMU_W BIT(1) +#define Z_XTENSA_MMU_XW (BIT(1) | BIT(0)) + #define Z_XTENSA_MMU_CACHED_WB BIT(2) #define Z_XTENSA_MMU_CACHED_WT BIT(3) + +#define K_MEM_PARTITION_IS_EXECUTABLE(attr) (((attr) & Z_XTENSA_MMU_X) != 0) +#define K_MEM_PARTITION_IS_WRITABLE(attr) (((attr) & Z_XENSA_MMU_W) != 0) + +/* Read-Write access permission attributes */ +#define K_MEM_PARTITION_P_RW_U_RW ((k_mem_partition_attr_t) \ + {Z_XTENSA_MMU_W}) +#define K_MEM_PARTITION_P_RW_U_NA ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_RO_U_RO ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_RO_U_NA ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_NA_U_NA ((k_mem_partition_attr_t) \ + {0}) + +/* Execution-allowed attributes */ +#define K_MEM_PARTITION_P_RX_U_RX ((k_mem_partition_attr_t) \ + {Z_XTENSA_MMU_X}) + +/* + * This BIT tells the mapping code whether the uncached pointer should + * be shared between all threads. That is not used in the HW, it is + * just for the implementation. + * + * The pte mapping this memory will use an ASID that is set in the + * ring 4 spot in RASID. + */ +#define Z_XTENSA_MMU_MAP_SHARED BIT(30) + #define Z_XTENSA_MMU_ILLEGAL (BIT(3) | BIT(2)) /* Struct used to map a memory region */ @@ -21,6 +53,8 @@ struct xtensa_mmu_range { const uint32_t attrs; }; +typedef uint32_t k_mem_partition_attr_t; + extern const struct xtensa_mmu_range xtensa_soc_mmu_ranges[]; extern int xtensa_soc_mmu_ranges_num;