xtensa: Enable userspace

Userspace support for Xtensa architecture using Xtensa MMU.

Some considerations:

- Syscalls are not inline functions like in other architectures because
  some compiler issues when using multiple registers to pass parameters
  to the syscall. So here we have a function call so we can use
  registers as we need.
- TLS is not supported by xcc in xtensa and reading PS register is
  a privileged instruction. So, we have to use threadptr to know if a
  thread is an user mode thread.

Signed-off-by: Flavio Ceolin <flavio.ceolin@intel.com>
Signed-off-by: Daniel Leung <daniel.leung@intel.com>
This commit is contained in:
Flavio Ceolin 2022-12-14 00:35:36 -08:00 committed by Carles Cufí
commit a651862b30
19 changed files with 1646 additions and 68 deletions

View file

@ -125,6 +125,7 @@ config XTENSA
select IRQ_OFFLOAD_NESTED if IRQ_OFFLOAD
select ARCH_HAS_CODE_DATA_RELOCATION
select ARCH_HAS_TIMING_FUNCTIONS
select ARCH_MEM_DOMAIN_DATA if USERSPACE
imply ATOMIC_OPERATIONS_ARCH
help
Xtensa architecture

View file

@ -113,6 +113,7 @@ config XTENSA_MMU
bool "Xtensa MMU Support"
default n
select MMU
select ARCH_MEM_DOMAIN_SYNCHRONOUS_API if USERSPACE
select XTENSA_SMALL_VECTOR_TABLE_ENTRY
select KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK if XTENSA_RPO_CACHE
help
@ -144,8 +145,18 @@ if XTENSA_MMU
The bit shift number for the virtual address for Xtensa
page table (PTEVADDR).
config XTENSA_MMU_NUM_L1_TABLES
int "Number of L1 page tables"
default 1 if !USERSPACE
default 4
help
This option specifies the maximum number of traslation tables.
Translation tables are directly related to the number of
memory domains in the target, considering the kernel itself requires one.
config XTENSA_MMU_NUM_L2_TABLES
int "Number of L2 page tables"
default 20 if USERSPACE
default 10
help
Each table can address up to 4MB memory address.
@ -159,6 +170,15 @@ if XTENSA_MMU
endif # XTENSA_MMU
config XTENSA_SYSCALL_USE_HELPER
bool "Use userspace syscall helper"
default y if "$(ZEPHYR_TOOLCHAIN_VARIANT)" = "xcc-clang"
depends on USERSPACE
help
Use syscall helpers for passing more then 3 arguments.
This is a workaround for toolchains where they have
issue modeling register usage.
endif # CPU_HAS_MMU
endmenu

View file

@ -22,6 +22,8 @@ zephyr_library_sources_ifdef(CONFIG_DEBUG_COREDUMP coredump.c)
zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c)
zephyr_library_sources_ifdef(CONFIG_GDBSTUB gdbstub.c)
zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU xtensa_mmu.c)
zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace.S)
zephyr_library_sources_ifdef(CONFIG_XTENSA_SYSCALL_USE_HELPER syscall_helper.c)
zephyr_library_sources_ifdef(
CONFIG_KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK

View file

@ -15,6 +15,7 @@
#endif
#endif
#include <zephyr/debug/coredump.h>
#include <zephyr/arch/common/exc_handle.h>
#include <zephyr/logging/log.h>
LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
@ -120,6 +121,14 @@ void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf)
z_fatal_error(reason, esf);
}
#ifdef CONFIG_USERSPACE
Z_EXC_DECLARE(z_xtensa_user_string_nlen);
static const struct z_exc_handle exceptions[] = {
Z_EXC_HANDLE(z_xtensa_user_string_nlen)
};
#endif /* CONFIG_USERSPACE */
#ifdef XT_SIMULATOR
void exit(int return_code)
{

View file

@ -18,18 +18,37 @@
#define Z_XTENSA_PTE_VPN_MASK 0xFFFFF000U
#define Z_XTENSA_PTE_PPN_MASK 0xFFFFF000U
#define Z_XTENSA_PTE_ATTR_MASK 0x0000000FU
#define Z_XTENSA_PTE_ATTR_CACHED_MASK 0x0000000CU
#define Z_XTENSA_L1_MASK 0x3FF00000U
#define Z_XTENSA_L2_MASK 0x3FFFFFU
#define Z_XTENSA_PPN_SHIFT 12U
#define Z_XTENSA_PTE_RING_MASK 0x00000030U
#define Z_XTENSA_PTE_RING_SHIFT 4U
#define Z_XTENSA_PTE(paddr, ring, attr) \
(((paddr) & Z_XTENSA_PTE_PPN_MASK) | \
(((ring) << 4) & Z_XTENSA_PTE_RING_MASK) | \
(((ring) << Z_XTENSA_PTE_RING_SHIFT) & Z_XTENSA_PTE_RING_MASK) | \
((attr) & Z_XTENSA_PTE_ATTR_MASK))
#define Z_XTENSA_PTE_ATTR_GET(pte) \
(pte) & Z_XTENSA_PTE_ATTR_MASK
#define Z_XTENSA_PTE_ATTR_SET(pte, attr) \
(((pte) & ~Z_XTENSA_PTE_ATTR_MASK) | (attr))
#define Z_XTENSA_PTE_RING_SET(pte, ring) \
(((pte) & ~Z_XTENSA_PTE_RING_MASK) | \
((ring) << Z_XTENSA_PTE_RING_SHIFT))
#define Z_XTENSA_PTE_RING_GET(pte) \
(((pte) & ~Z_XTENSA_PTE_RING_MASK) >> Z_XTENSA_PTE_RING_SHIFT)
#define Z_XTENSA_PTE_ASID_GET(pte, rasid) \
(((rasid) >> ((((pte) & Z_XTENSA_PTE_RING_MASK) \
>> Z_XTENSA_PTE_RING_SHIFT) * 8)) & 0xFF)
#define Z_XTENSA_TLB_ENTRY(vaddr, way) \
(((vaddr) & Z_XTENSA_PTE_PPN_MASK) | (way))
@ -38,11 +57,38 @@
(((vaddr) >> Z_XTENSA_PPN_SHIFT) & 0x03U))
#define Z_XTENSA_L2_POS(vaddr) \
(((vaddr) & Z_XTENSA_L2_MASK) >> Z_XTENSA_PPN_SHIFT)
(((vaddr) & Z_XTENSA_L2_MASK) >> 12U)
#define Z_XTENSA_L1_POS(vaddr) \
((vaddr) >> 22U)
/* PTE attributes for entries in the L1 page table. Should never be
* writable, may be cached in non-SMP contexts only
*/
#if CONFIG_MP_MAX_NUM_CPUS == 1
#define Z_XTENSA_PAGE_TABLE_ATTR Z_XTENSA_MMU_CACHED_WB
#else
#define Z_XTENSA_PAGE_TABLE_ATTR 0
#endif
/* This ASID is shared between all domains and kernel. */
#define Z_XTENSA_MMU_SHARED_ASID 255
/* Fixed data TLB way to map the page table */
#define Z_XTENSA_MMU_PTE_WAY 7
/* Fixed data TLB way to map the vecbase */
#define Z_XTENSA_MMU_VECBASE_WAY 8
/* Kernel specific ASID. Ring field in the PTE */
#define Z_XTENSA_KERNEL_RING 0
/* User specific ASID. Ring field in the PTE */
#define Z_XTENSA_USER_RING 2
/* Ring value for MMU_SHARED_ASID */
#define Z_XTENSA_SHARED_RING 3
/* Number of data TLB ways [0-9] */
#define Z_XTENSA_DTLB_WAYS 10
@ -96,6 +142,14 @@
#define Z_XTENSA_PAGE_TABLE_VADDR \
Z_XTENSA_PTE_ENTRY_VADDR(Z_XTENSA_PTEVADDR)
/*
* Get asid for a given ring from rasid register.
* rasid contains four asid, one per ring.
*/
#define Z_XTENSA_RASID_ASID_GET(rasid, ring) \
(((rasid) >> ((ring) * 8)) & 0xff)
static ALWAYS_INLINE void xtensa_rasid_set(uint32_t rasid)
{
__asm__ volatile("wsr %0, rasid\n\t"
@ -110,6 +164,16 @@ static ALWAYS_INLINE uint32_t xtensa_rasid_get(void)
return rasid;
}
static ALWAYS_INLINE void xtensa_rasid_asid_set(uint8_t asid, uint8_t pos)
{
uint32_t rasid = xtensa_rasid_get();
rasid = (rasid & ~(0xff << (pos * 8))) | ((uint32_t)asid << (pos * 8));
xtensa_rasid_set(rasid);
}
static ALWAYS_INLINE void xtensa_itlb_entry_invalidate(uint32_t entry)
{
__asm__ volatile("iitlb %0\n\t"
@ -201,6 +265,21 @@ static ALWAYS_INLINE void xtensa_ptevaddr_set(void *ptables)
__asm__ volatile("wsr.ptevaddr %0" : : "a"((uint32_t)ptables));
}
/**
* @brief Get the current page tables.
*
* The page tables is obtained by reading ptevaddr address.
*
* @return ptables The page tables address (virtual address)
*/
static ALWAYS_INLINE void *xtensa_ptevaddr_get(void)
{
uint32_t ptables;
__asm__ volatile("rsr.ptevaddr %0" : "=a" (ptables));
return (void *)ptables;
}
/*
* The following functions are helpful when debugging.
*/

View file

@ -5,6 +5,7 @@
#include <gen_offset.h>
#include <kernel_offsets.h>
#include <zephyr/arch/xtensa/thread.h>
#include <xtensa-asm2-context.h>
@ -60,4 +61,10 @@ GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu14);
GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu15);
#endif
#ifdef CONFIG_USERSPACE
GEN_OFFSET_SYM(_thread_arch_t, psp);
GEN_OFFSET_SYM(_thread_arch_t, ptables);
#endif
GEN_ABS_SYM_END

View file

@ -0,0 +1,124 @@
/*
* Copyright (c) 2022 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/arch/xtensa/syscall.h>
uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t arg5, uintptr_t arg6,
uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
register uintptr_t a5 __asm__("%a5") = arg4;
register uintptr_t a8 __asm__("%a8") = arg5;
register uintptr_t a9 __asm__("%a9") = arg6;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4),
"r" (a5), "r" (a8), "r" (a9)
: "memory");
return a2;
}
uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t arg5, uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
register uintptr_t a5 __asm__("%a5") = arg4;
register uintptr_t a8 __asm__("%a8") = arg5;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4),
"r" (a5), "r" (a8)
: "memory");
return a2;
}
uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
register uintptr_t a5 __asm__("%a5") = arg4;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4),
"r" (a5)
: "memory");
return a2;
}
uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4)
: "memory");
return a2;
}
uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3)
: "memory");
return a2;
}
uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6)
: "memory");
return a2;
}
uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id)
{
register uintptr_t a2 __asm__("%a2") = call_id;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2)
: "memory");
return a2;
}

View file

@ -0,0 +1,302 @@
/*
* Copyright (c) 2022, Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <xtensa-asm2-s.h>
#include <offsets.h>
#include <offsets_short.h>
#include <zephyr/syscall.h>
#include <zsr.h>
/**
* syscall number arg1, arg2, arg3, arg4, arg5, arg6
* -------------- ----------------------------------
* a2 a6, a3, a4, a5, a8, a9
*
**/
.pushsection .text.z_xtensa_do_syscall, "ax"
.global z_xtensa_do_syscall
.align 4
z_xtensa_do_syscall:
rsr a0, ZSR_CPU
l32i a0, a0, ___cpu_t_current_OFFSET
l32i a0, a0, _thread_offset_to_psp
addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF
s32i a1, a0, ___xtensa_irq_bsa_t_scratch_OFFSET
s32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET
s32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET
rsr a2, ZSR_A0SAVE
s32i a2, a0, ___xtensa_irq_bsa_t_a0_OFFSET
rsr.ps a2
movi a3, ~PS_OWB_MASK
and a2, a2, a3
s32i a2, a0, ___xtensa_irq_bsa_t_ps_OFFSET
rsr.epc1 a2
s32i a2, a0, ___xtensa_irq_bsa_t_pc_OFFSET
movi a2, PS_WOE|PS_INTLEVEL(XCHAL_NMILEVEL)
rsr.ps a3
or a3, a3, a2
movi a2, ~(PS_EXCM | PS_RING_MASK)
and a3, a3, a2
wsr.ps a3
rsync
l32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET
l32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET
SPILL_ALL_WINDOWS
rsr a0, ZSR_CPU
l32i a0, a0, ___cpu_t_current_OFFSET
l32i a0, a0, _thread_offset_to_psp
addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF
mov a1, a0
l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET
#if XCHAL_HAVE_LOOPS
/* If the syscall instruction was the last instruction in the body of
* a zero-overhead loop, and the loop will execute again, decrement
* the loop count and resume execution at the head of the loop.
*/
rsr.lend a2
addi a3, a3, 3
bne a2, a3, end_loop
rsr.lcount a2
beqz a2, end_loop
addi a2, a2, -1
wsr.lcount a2
rsr.lbeg a3
end_loop:
#else
/* EPC1 (and now a3) contains the address that invoked syscall.
* We need to increment it to execute the next instruction when
* we return. The instruction size is 3 bytes, so lets just add it.
*/
addi a3, a3, 3
#endif
s32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET
ODD_REG_SAVE
call0 xtensa_save_high_regs
l32i a2, a1, 0
l32i a2, a2, ___xtensa_irq_bsa_t_a2_OFFSET
movi a0, K_SYSCALL_LIMIT
bgeu a2, a0, _bad_syscall
_id_ok:
/* Find the function handler for the given syscall id. */
movi a3, _k_syscall_table
slli a2, a2, 2
add a2, a2, a3
l32i a2, a2, 0
/* Clear up the threadptr because it is used
* to check if a thread is running on user mode. Since
* we are in a interruption we don't want the system
* thinking it is possibly running in user mode.
*/
movi a0, 0
wur.THREADPTR a0
/* Set syscall parameters. We have an initial call4 to set up the
* the stack and then a new call4 for the syscall function itself.
* So parameters should be put as if it was a call8.
*/
mov a10, a8
mov a11, a9
mov a8, a4
mov a9, a5
l32i a3, a1, 0
l32i a7, a3, ___xtensa_irq_bsa_t_a3_OFFSET
/* Since we are unmasking EXCM, we need to set RING bits to kernel
* mode, otherwise we won't be able to run the exception handler in C.
*/
movi a0, PS_WOE|PS_CALLINC(0)|PS_UM|PS_INTLEVEL(0)
wsr.ps a0
rsync
call4 _syscall_call0
/* copy return value. Lets put it in the top of stack
* because registers will be clobbered in
* xtensa_restore_high_regs
*/
l32i a3, a1, 0
s32i a6, a3, ___xtensa_irq_bsa_t_a2_OFFSET
j _syscall_returned
.align 4
_syscall_call0:
/* We want an ENTRY to set a bit in windowstart */
jx a2
_syscall_returned:
call0 xtensa_restore_high_regs
l32i a3, a1, ___xtensa_irq_bsa_t_sar_OFFSET
wsr a3, SAR
#if XCHAL_HAVE_LOOPS
l32i a3, a1, ___xtensa_irq_bsa_t_lbeg_OFFSET
wsr a3, LBEG
l32i a3, a1, ___xtensa_irq_bsa_t_lend_OFFSET
wsr a3, LEND
l32i a3, a1, ___xtensa_irq_bsa_t_lcount_OFFSET
wsr a3, LCOUNT
#endif
#if XCHAL_HAVE_S32C1I
l32i a3, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET
wsr a3, SCOMPARE1
#endif
rsr a3, ZSR_CPU
l32i a3, a3, ___cpu_t_current_OFFSET
wur.THREADPTR a3
l32i a3, a1, ___xtensa_irq_bsa_t_ps_OFFSET
wsr.ps a3
l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET
wsr.epc1 a3
l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET
l32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET
l32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET
l32i a1, a1, ___xtensa_irq_bsa_t_scratch_OFFSET
rsync
rfe
_bad_syscall:
movi a2, K_SYSCALL_BAD
j _id_ok
.popsection
/* FUNC_NORETURN void z_xtensa_userspace_enter(k_thread_entry_t user_entry,
* void *p1, void *p2, void *p3,
* uint32_t stack_end,
* uint32_t stack_start)
*
* A one-way trip to userspace.
*/
.global z_xtensa_userspace_enter
.type z_xtensa_userspace_enter, @function
.align 4
z_xtensa_userspace_enter:
/* Call entry to set a bit in the windowstart and
* do the rotation, but we are going to set our own
* stack.
*/
entry a1, 16
/* We have to switch to kernel stack before spill kernel data and
* erase user stack to avoid leak from previous context.
*/
mov a1, a7 /* stack start (low address) */
addi a1, a1, -16
SPILL_ALL_WINDOWS
rsr a0, ZSR_CPU
l32i a0, a0, ___cpu_t_current_OFFSET
addi a1, a1, -28
s32i a0, a1, 24
s32i a2, a1, 20
s32i a3, a1, 16
s32i a4, a1, 12
s32i a5, a1, 8
s32i a6, a1, 4
s32i a7, a1, 0
l32i a6, a1, 24
call4 xtensa_user_stack_perms
l32i a6, a1, 24
call4 z_xtensa_swap_update_page_tables
/* Set threadptr with the thread address, we are going to user mode. */
l32i a0, a1, 24
wur.THREADPTR a0
/* Set now z_thread_entry parameters, we are simulating a call4
* call, so parameters start at a6, a7, ...
*/
l32i a6, a1, 20
l32i a7, a1, 16
l32i a8, a1, 12
l32i a9, a1, 8
/* stash user stack */
l32i a0, a1, 4
addi a1, a1, 28
/* Go back to user stack */
mov a1, a0
movi a0, z_thread_entry
wsr.epc2 a0
/* Configuring PS register.
* We have to set callinc as well, since the called
* function will do "entry"
*/
movi a0, PS_WOE|PS_CALLINC(1)|PS_UM|PS_RING(2)
wsr a0, EPS2
movi a0, 0
rfi 2
/*
* size_t arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg)
*/
.global arch_user_string_nlen
.type arch_user_string_nlen, @function
.align 4
arch_user_string_nlen:
entry a1, 32
/* error value, set to -1. */
movi a5, -1
s32i a5, a4, 0
/* length count */
xor a5, a5, a5
/* This code might page fault */
strlen_loop:
.global z_xtensa_user_string_nlen_fault_start
z_xtensa_user_string_nlen_fault_start:
l8ui a6, a2, 0 /* Current char */
.global z_xtensa_user_string_nlen_fault_end
z_xtensa_user_string_nlen_fault_end:
beqz a6, strlen_done
addi a5, a5, 1
addi a2, a2, 1
beq a5, a3, strlen_done
j strlen_loop
strlen_done:
/* Set return value */
mov a2, a5
/* Set error value to 0 since we succeeded */
movi a5, 0x0
s32i a5, a4, 0
.global z_xtensa_user_string_nlen_fixup
z_xtensa_user_string_nlen_fixup:
retw

View file

@ -174,7 +174,8 @@ _restore_context:
l32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET
wsr a0, SCOMPARE1
#endif
#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE)
#if XCHAL_HAVE_THREADPTR && \
(defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE))
l32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET
wur a0, THREADPTR
#endif
@ -258,6 +259,16 @@ noflush:
l32i a3, a2, ___xtensa_irq_bsa_t_a3_OFFSET
s32i a1, a3, 0
#ifdef CONFIG_USERSPACE
/* Switch page tables */
rsr a6, ZSR_CPU
l32i a6, a6, ___cpu_t_current_OFFSET
call4 z_xtensa_swap_update_page_tables
l32i a2, a3, 0
l32i a2, a2, 0
#endif
/* Switch stack pointer and restore. The jump to
* _restore_context does not return as such, but we arrange
* for the restored "next" address to be immediately after for
@ -347,6 +358,9 @@ _Level1RealVector:
rsr.exccause a0
#ifdef CONFIG_XTENSA_MMU
beqi a0, EXCCAUSE_ITLB_MISS, _handle_tlb_miss_user
#ifdef CONFIG_USERSPACE
beqi a0, EXCCAUSE_SYSCALL, _syscall
#endif /* CONFIG_USERSPACE */
addi a0, a0, -EXCCAUSE_DTLB_MISS
beqz a0, _handle_tlb_miss_user
rsr.exccause a0
@ -376,6 +390,11 @@ _handle_tlb_miss_user:
l32i a0, a0, 0
rsr a0, ZSR_A0SAVE
rfe
#ifdef CONFIG_USERSPACE
_syscall:
rsr a0, ZSR_A0SAVE
j z_xtensa_do_syscall
#endif /* CONFIG_USERSPACE */
#endif /* CONFIG_XTENSA_MMU */
.popsection

View file

@ -26,6 +26,13 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top,
{
void *ret;
_xtensa_irq_stack_frame_a11_t *frame;
#ifdef CONFIG_USERSPACE
struct z_xtensa_thread_stack_header *header =
(struct z_xtensa_thread_stack_header *)thread->stack_obj;
thread->arch.psp = header->privilege_stack +
sizeof(header->privilege_stack);
#endif
/* Not-a-cpu ID Ensures that the first time this is run, the
* stack will be invalidated. That covers the edge case of
@ -48,11 +55,23 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top,
(void)memset(frame, 0, bsasz);
frame->bsa.pc = (uintptr_t)z_thread_entry;
frame->bsa.ps = PS_WOE | PS_UM | PS_CALLINC(1);
#ifdef CONFIG_USERSPACE
if ((thread->base.user_options & K_USER) == K_USER) {
frame->bsa.pc = (uintptr_t)arch_user_mode_enter;
} else {
frame->bsa.pc = (uintptr_t)z_thread_entry;
}
#else
frame->bsa.pc = (uintptr_t)z_thread_entry;
#endif
#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE)
#if XCHAL_HAVE_THREADPTR
#ifdef CONFIG_THREAD_LOCAL_STORAGE
frame->bsa.threadptr = thread->tls;
#elif CONFIG_USERSPACE
frame->bsa.threadptr = (uintptr_t)((thread->base.user_options & K_USER) ? thread : NULL);
#endif
#endif
/* Arguments to z_thread_entry(). Remember these start at A6,
@ -471,3 +490,24 @@ void arch_spin_relax(void)
#undef NOP1
}
#endif /* CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS */
#ifdef CONFIG_USERSPACE
FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
void *p1, void *p2, void *p3)
{
struct k_thread *current = _current;
size_t stack_end;
/* Transition will reset stack pointer to initial, discarding
* any old context since this is a one-way operation
*/
stack_end = Z_STACK_PTR_ALIGN(current->stack_info.start +
current->stack_info.size -
current->stack_info.delta);
z_xtensa_userspace_enter(user_entry, p1, p2, p3,
stack_end, current->stack_info.start);
CODE_UNREACHABLE;
}
#endif /* CONFIG_USERSPACE */

View file

@ -4,6 +4,7 @@
*/
#include <zephyr/kernel.h>
#include <zephyr/cache.h>
#include <zephyr/arch/xtensa/arch.h>
#include <zephyr/arch/xtensa/xtensa_mmu.h>
#include <zephyr/linker/linker-defs.h>
#include <zephyr/logging/log.h>
@ -15,22 +16,24 @@
#include <kernel_arch_func.h>
#include <mmu.h>
/* Fixed data TLB way to map the page table */
#define MMU_PTE_WAY 7
/* Fixed data TLB way to map VECBASE */
#define MMU_VECBASE_WAY 8
/* Level 1 contains page table entries
* necessary to map the page table itself.
*/
#define XTENSA_L1_PAGE_TABLE_ENTRIES 1024U
/* Size of level 1 page table.
*/
#define XTENSA_L1_PAGE_TABLE_SIZE (XTENSA_L1_PAGE_TABLE_ENTRIES * sizeof(uint32_t))
/* Level 2 contains page table entries
* necessary to map the page table itself.
*/
#define XTENSA_L2_PAGE_TABLE_ENTRIES 1024U
/* Size of level 2 page table.
*/
#define XTENSA_L2_PAGE_TABLE_SIZE (XTENSA_L2_PAGE_TABLE_ENTRIES * sizeof(uint32_t))
LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000,
@ -40,8 +43,18 @@ BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000,
* Level 1 page table has to be 4Kb to fit into one of the wired entries.
* All entries are initialized as INVALID, so an attempt to read an unmapped
* area will cause a double exception.
*
* Each memory domain contains its own l1 page table. The kernel l1 page table is
* located at the index 0.
*/
uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4));
static uint32_t l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES]
__aligned(KB(4));
/*
* That is an alias for the page tables set used by the kernel.
*/
uint32_t *z_xtensa_kernel_ptables = (uint32_t *)l1_page_table[0];
/*
* Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one
@ -50,12 +63,41 @@ uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4));
static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_TABLE_ENTRIES]
__aligned(KB(4));
/*
* This additional variable tracks which l1 tables are in use. This is kept separated from
* the tables to keep alignment easier.
*
* @note: The first bit is set because it is used for the kernel page tables.
*/
static ATOMIC_DEFINE(l1_page_table_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES);
/*
* This additional variable tracks which l2 tables are in use. This is kept separated from
* the tables to keep alignment easier.
*/
static ATOMIC_DEFINE(l2_page_tables_track, CONFIG_XTENSA_MMU_NUM_L2_TABLES);
/*
* Protects xtensa_domain_list and serializes access to page tables.
*/
static struct k_spinlock xtensa_mmu_lock;
#ifdef CONFIG_USERSPACE
/*
* Each domain has its own ASID. ASID can go through 1 (kernel) to 255.
* When a TLB entry matches, the hw will check the ASID in the entry and finds
* the correspondent position in the RASID register. This position will then be
* compared with the current ring (CRING) to check the permission.
*/
static uint8_t asid_count = 3;
/*
* List with all active and initialized memory domains.
*/
static sys_slist_t xtensa_domain_list;
#endif /* CONFIG_USERSPACE */
extern char _heap_end[];
extern char _heap_start[];
extern char __data_start[];
@ -100,18 +142,61 @@ static const struct xtensa_mmu_range mmu_zephyr_ranges[] = {
{
.start = (uint32_t)__text_region_start,
.end = (uint32_t)__text_region_end,
.attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB,
.attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED,
.name = "text",
},
/* Mark rodata segment cacheable, read only and non-executable */
{
.start = (uint32_t)__rodata_region_start,
.end = (uint32_t)__rodata_region_end,
.attrs = Z_XTENSA_MMU_CACHED_WB,
.attrs = Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED,
.name = "rodata",
},
};
static inline uint32_t *thread_page_tables_get(const struct k_thread *thread)
{
#ifdef CONFIG_USERSPACE
if ((thread->base.user_options & K_USER) != 0U) {
return thread->arch.ptables;
}
#endif
return z_xtensa_kernel_ptables;
}
/**
* @brief Check if the page table entry is illegal.
*
* @param[in] Page table entry.
*/
static inline bool is_pte_illegal(uint32_t pte)
{
uint32_t attr = pte & Z_XTENSA_PTE_ATTR_MASK;
/*
* The ISA manual states only 12 and 14 are illegal values.
* 13 and 15 are not. So we need to be specific than simply
* testing if bits 2 and 3 are set.
*/
return (attr == 12) || (attr == 14);
}
/*
* @brief Initialize all page table entries to be illegal.
*
* @param[in] Pointer to page table.
* @param[in] Number of page table entries in the page table.
*/
static void init_page_table(uint32_t *ptable, size_t num_entries)
{
int i;
for (i = 0; i < num_entries; i++) {
ptable[i] = Z_XTENSA_MMU_ILLEGAL;
}
}
static inline uint32_t *alloc_l2_table(void)
{
uint16_t idx;
@ -125,45 +210,86 @@ static inline uint32_t *alloc_l2_table(void)
return NULL;
}
/**
* @brief Switch page tables
*
* This switches the page tables to the incoming ones (@a ptables).
* Since data TLBs to L2 page tables are auto-filled, @a dtlb_inv
* can be used to invalidate these data TLBs. @a cache_inv can be
* set to true to invalidate cache to the page tables.
*
* @param[in] ptables Page tables to be switched to.
* @param[in] dtlb_inv True if to invalidate auto-fill data TLBs.
* @param[in] cache_inv True if to invalidate cache to page tables.
*/
static ALWAYS_INLINE void switch_page_tables(uint32_t *ptables, bool dtlb_inv, bool cache_inv)
{
if (cache_inv) {
sys_cache_data_invd_range((void *)ptables, XTENSA_L1_PAGE_TABLE_SIZE);
sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables));
}
/* Invalidate data TLB to L1 page table */
xtensa_dtlb_vaddr_invalidate((void *)Z_XTENSA_PAGE_TABLE_VADDR);
/* Now map the pagetable itself with KERNEL asid to avoid user thread
* from tampering with it.
*/
xtensa_dtlb_entry_write_sync(
Z_XTENSA_PTE((uint32_t)ptables, Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR),
Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY));
if (dtlb_inv) {
/* Since L2 page tables are auto-refilled,
* invalidate all of them to flush the old entries out.
*/
xtensa_tlb_autorefill_invalidate();
}
}
static void map_memory_range(const uint32_t start, const uint32_t end,
const uint32_t attrs)
const uint32_t attrs, bool shared)
{
uint32_t page, *table;
for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) {
uint32_t pte = Z_XTENSA_PTE(page, Z_XTENSA_KERNEL_RING, attrs);
uint32_t pte = Z_XTENSA_PTE(page,
shared ? Z_XTENSA_SHARED_RING : Z_XTENSA_KERNEL_RING,
attrs);
uint32_t l2_pos = Z_XTENSA_L2_POS(page);
uint32_t l1_pos = page >> 22;
uint32_t l1_pos = Z_XTENSA_L1_POS(page);
if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) {
if (is_pte_illegal(z_xtensa_kernel_ptables[l1_pos])) {
table = alloc_l2_table();
__ASSERT(table != NULL, "There is no l2 page table available to "
"map 0x%08x\n", page);
l1_page_table[l1_pos] =
init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES);
z_xtensa_kernel_ptables[l1_pos] =
Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING,
Z_XTENSA_MMU_CACHED_WT);
Z_XTENSA_PAGE_TABLE_ATTR);
}
table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
table = (uint32_t *)(z_xtensa_kernel_ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
table[l2_pos] = pte;
}
}
static void map_memory(const uint32_t start, const uint32_t end,
const uint32_t attrs)
const uint32_t attrs, bool shared)
{
map_memory_range(start, end, attrs);
map_memory_range(start, end, attrs, shared);
#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP
if (arch_xtensa_is_ptr_uncached((void *)start)) {
map_memory_range(POINTER_TO_UINT(z_soc_cached_ptr((void *)start)),
POINTER_TO_UINT(z_soc_cached_ptr((void *)end)),
attrs | Z_XTENSA_MMU_CACHED_WB);
attrs | Z_XTENSA_MMU_CACHED_WB, shared);
} else if (arch_xtensa_is_ptr_cached((void *)start)) {
map_memory_range(POINTER_TO_UINT(z_soc_uncached_ptr((void *)start)),
POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs);
POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs, shared);
}
#endif
}
@ -171,16 +297,19 @@ static void map_memory(const uint32_t start, const uint32_t end,
static void xtensa_init_page_tables(void)
{
volatile uint8_t entry;
uint32_t page;
for (page = 0; page < XTENSA_L1_PAGE_TABLE_ENTRIES; page++) {
l1_page_table[page] = Z_XTENSA_MMU_ILLEGAL;
}
init_page_table(z_xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES);
atomic_set_bit(l1_page_table_track, 0);
for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) {
const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry];
bool shared;
uint32_t attrs;
map_memory(range->start, range->end, range->attrs);
shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED);
attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED;
map_memory(range->start, range->end, attrs, shared);
}
/**
@ -198,8 +327,13 @@ static void xtensa_init_page_tables(void)
#endif
for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) {
const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry];
bool shared;
uint32_t attrs;
map_memory(range->start, range->end, range->attrs);
shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED);
attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED;
map_memory(range->start, range->end, attrs, shared);
}
#if defined(__GNUC__)
#pragma GCC diagnostic pop
@ -231,6 +365,9 @@ void z_xtensa_mmu_init(void)
/* Set the page table location in the virtual address */
xtensa_ptevaddr_set((void *)Z_XTENSA_PTEVADDR);
/* Set rasid */
xtensa_rasid_asid_set(Z_XTENSA_MMU_SHARED_ASID, Z_XTENSA_SHARED_RING);
/* Next step is to invalidate the tlb entry that contains the top level
* page table. This way we don't cause a multi hit exception.
*/
@ -243,9 +380,9 @@ void z_xtensa_mmu_init(void)
* Lets use one of the wired entry, so we never have tlb miss for
* the top level table.
*/
xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)l1_page_table, Z_XTENSA_KERNEL_RING,
Z_XTENSA_MMU_CACHED_WT),
Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, MMU_PTE_WAY));
xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)z_xtensa_kernel_ptables,
Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR),
Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY));
/* Before invalidate the text region in the TLB entry 6, we need to
* map the exception vector into one of the wired entries to avoid
@ -297,7 +434,7 @@ void z_xtensa_mmu_init(void)
xtensa_dtlb_entry_write(
Z_XTENSA_PTE((uint32_t)vecbase,
Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_CACHED_WB),
Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, MMU_VECBASE_WAY));
Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, Z_XTENSA_MMU_VECBASE_WAY));
/*
* Pre-load TLB for vecbase so exception handling won't result
@ -325,6 +462,12 @@ void z_xtensa_mmu_init(void)
xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3));
xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3));
/*
* Clear out THREADPTR as we use it to indicate
* whether we are in user mode or not.
*/
XTENSA_WUR("THREADPTR", 0);
arch_xtensa_mmu_post_init(_current_cpu->id == 0);
}
@ -351,32 +494,121 @@ __weak void arch_reserved_pages_update(void)
}
#endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */
static bool l2_page_table_map(void *vaddr, uintptr_t phys, uint32_t flags)
static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys,
uint32_t flags, bool is_user)
{
uint32_t l1_pos = (uint32_t)vaddr >> 22;
uint32_t pte = Z_XTENSA_PTE(phys, Z_XTENSA_KERNEL_RING, flags);
uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr);
uint32_t *table;
if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) {
sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
if (is_pte_illegal(l1_table[l1_pos])) {
table = alloc_l2_table();
if (table == NULL) {
return false;
}
l1_page_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING,
Z_XTENSA_MMU_CACHED_WT);
init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES);
l1_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING,
Z_XTENSA_PAGE_TABLE_ATTR);
sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
}
table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
table[l2_pos] = pte;
table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
table[l2_pos] = Z_XTENSA_PTE(phys, is_user ? Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING,
flags);
if ((flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) {
sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0]));
return true;
}
static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, bool is_user)
{
bool ret;
void *vaddr, *vaddr_uc;
uintptr_t paddr, paddr_uc;
uint32_t flags, flags_uc;
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
if (arch_xtensa_is_ptr_cached(va)) {
vaddr = va;
vaddr_uc = arch_xtensa_uncached_ptr(va);
} else {
vaddr = arch_xtensa_cached_ptr(va);
vaddr_uc = va;
}
if (arch_xtensa_is_ptr_cached((void *)pa)) {
paddr = pa;
paddr_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)pa);
} else {
paddr = (uintptr_t)arch_xtensa_cached_ptr((void *)pa);
paddr_uc = pa;
}
flags_uc = (xtensa_flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK);
flags = flags_uc | Z_XTENSA_MMU_CACHED_WB;
} else {
vaddr = va;
paddr = pa;
flags = xtensa_flags;
}
ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr, paddr,
flags, is_user);
__ASSERT(ret, "Virtual address (%p) already mapped", va);
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) {
ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc,
flags_uc, is_user);
__ASSERT(ret, "Virtual address (%p) already mapped", vaddr_uc);
}
#ifndef CONFIG_USERSPACE
ARG_UNUSED(ret);
#else
if (ret) {
sys_snode_t *node;
struct arch_mem_domain *domain;
k_spinlock_key_t key;
key = k_spin_lock(&z_mem_domain_lock);
SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) {
domain = CONTAINER_OF(node, struct arch_mem_domain, node);
ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr,
flags, is_user);
__ASSERT(ret, "Virtual address (%p) already mapped for domain %p",
vaddr, domain);
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) {
ret = l2_page_table_map(domain->ptables,
(void *)vaddr_uc, paddr_uc,
flags_uc, is_user);
__ASSERT(ret, "Virtual address (%p) already mapped for domain %p",
vaddr_uc, domain);
}
}
k_spin_unlock(&z_mem_domain_lock, key);
}
#endif /* CONFIG_USERSPACE */
if ((xtensa_flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) {
xtensa_itlb_vaddr_invalidate(vaddr);
}
xtensa_dtlb_vaddr_invalidate(vaddr);
return true;
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
if (xtensa_flags & Z_XTENSA_MMU_X) {
xtensa_itlb_vaddr_invalidate(vaddr_uc);
}
xtensa_dtlb_vaddr_invalidate(vaddr_uc);
}
}
void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
@ -385,7 +617,8 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
uint32_t pa = (uint32_t)phys;
uint32_t rem_size = (uint32_t)size;
uint32_t xtensa_flags = 0;
int key;
k_spinlock_key_t key;
bool is_user;
if (size == 0) {
LOG_ERR("Cannot map physical memory at 0x%08X: invalid "
@ -414,63 +647,130 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
xtensa_flags |= Z_XTENSA_MMU_X;
}
key = arch_irq_lock();
is_user = (flags & K_MEM_PERM_USER) == K_MEM_PERM_USER;
key = k_spin_lock(&xtensa_mmu_lock);
while (rem_size > 0) {
bool ret = l2_page_table_map((void *)va, pa, xtensa_flags);
__arch_mem_map((void *)va, pa, xtensa_flags, is_user);
ARG_UNUSED(ret);
__ASSERT(ret, "Virtual address (%u) already mapped", (uint32_t)virt);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
pa += KB(4);
}
arch_irq_unlock(key);
k_spin_unlock(&xtensa_mmu_lock, key);
}
static void l2_page_table_unmap(void *vaddr)
/**
* @return True if page is executable (thus need to invalidate ITLB),
* false if not.
*/
static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr)
{
uint32_t l1_pos = (uint32_t)vaddr >> 22;
uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr);
uint32_t *table;
uint32_t *l2_table;
uint32_t table_pos;
bool exec;
if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) {
return;
sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
if (is_pte_illegal(l1_table[l1_pos])) {
/* We shouldn't be unmapping an illegal entry.
* Return true so that we can invalidate ITLB too.
*/
return true;
}
exec = l1_page_table[l1_pos] & Z_XTENSA_MMU_X;
exec = l1_table[l1_pos] & Z_XTENSA_MMU_X;
table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
table[l2_pos] = Z_XTENSA_MMU_ILLEGAL;
l2_table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
l2_table[l2_pos] = Z_XTENSA_MMU_ILLEGAL;
sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) {
if (table[l2_pos] != Z_XTENSA_MMU_ILLEGAL) {
if (!is_pte_illegal(l2_table[l2_pos])) {
goto end;
}
}
l1_page_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL;
table_pos = (table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES);
l1_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL;
sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0]));
table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES);
atomic_clear_bit(l2_page_tables_track, table_pos);
/* Need to invalidate L2 page table as it is no longer valid. */
xtensa_dtlb_vaddr_invalidate((void *)table);
xtensa_dtlb_vaddr_invalidate((void *)l2_table);
end:
if (exec) {
return exec;
}
static inline void __arch_mem_unmap(void *va)
{
bool is_exec;
void *vaddr, *vaddr_uc;
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
if (arch_xtensa_is_ptr_cached(va)) {
vaddr = va;
vaddr_uc = arch_xtensa_uncached_ptr(va);
} else {
vaddr = arch_xtensa_cached_ptr(va);
vaddr_uc = va;
}
} else {
vaddr = va;
}
is_exec = l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr);
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
(void)l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr_uc);
}
#ifdef CONFIG_USERSPACE
sys_snode_t *node;
struct arch_mem_domain *domain;
k_spinlock_key_t key;
key = k_spin_lock(&z_mem_domain_lock);
SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) {
domain = CONTAINER_OF(node, struct arch_mem_domain, node);
(void)l2_page_table_unmap(domain->ptables, (void *)vaddr);
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
(void)l2_page_table_unmap(domain->ptables, (void *)vaddr_uc);
}
}
k_spin_unlock(&z_mem_domain_lock, key);
#endif /* CONFIG_USERSPACE */
if (is_exec) {
xtensa_itlb_vaddr_invalidate(vaddr);
}
xtensa_dtlb_vaddr_invalidate(vaddr);
if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) {
if (is_exec) {
xtensa_itlb_vaddr_invalidate(vaddr_uc);
}
xtensa_dtlb_vaddr_invalidate(vaddr_uc);
}
}
void arch_mem_unmap(void *addr, size_t size)
{
uint32_t va = (uint32_t)addr;
uint32_t rem_size = (uint32_t)size;
int key;
k_spinlock_key_t key;
if (addr == NULL) {
LOG_ERR("Cannot unmap NULL pointer");
@ -482,13 +782,363 @@ void arch_mem_unmap(void *addr, size_t size)
return;
}
key = arch_irq_lock();
key = k_spin_lock(&xtensa_mmu_lock);
while (rem_size > 0) {
l2_page_table_unmap((void *)va);
__arch_mem_unmap((void *)va);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
}
arch_irq_unlock(key);
k_spin_unlock(&xtensa_mmu_lock, key);
}
#ifdef CONFIG_USERSPACE
static inline uint32_t *alloc_l1_table(void)
{
uint16_t idx;
for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L1_TABLES; idx++) {
if (!atomic_test_and_set_bit(l1_page_table_track, idx)) {
return (uint32_t *)&l1_page_table[idx];
}
}
return NULL;
}
static uint32_t *dup_table(uint32_t *source_table)
{
uint16_t i, j;
uint32_t *dst_table = alloc_l1_table();
if (!dst_table) {
return NULL;
}
for (i = 0; i < XTENSA_L1_PAGE_TABLE_ENTRIES; i++) {
uint32_t *l2_table, *src_l2_table;
if (is_pte_illegal(source_table[i])) {
dst_table[i] = Z_XTENSA_MMU_ILLEGAL;
continue;
}
src_l2_table = (uint32_t *)(source_table[i] & Z_XTENSA_PTE_PPN_MASK);
l2_table = alloc_l2_table();
if (l2_table == NULL) {
goto err;
}
for (j = 0; j < XTENSA_L2_PAGE_TABLE_ENTRIES; j++) {
l2_table[j] = src_l2_table[j];
}
/* The page table is using kernel ASID because we don't
* user thread manipulate it.
*/
dst_table[i] = Z_XTENSA_PTE((uint32_t)l2_table, Z_XTENSA_KERNEL_RING,
Z_XTENSA_PAGE_TABLE_ATTR);
sys_cache_data_flush_range((void *)l2_table, XTENSA_L2_PAGE_TABLE_SIZE);
}
sys_cache_data_flush_range((void *)dst_table, XTENSA_L1_PAGE_TABLE_SIZE);
return dst_table;
err:
/* TODO: Cleanup failed allocation*/
return NULL;
}
int arch_mem_domain_init(struct k_mem_domain *domain)
{
uint32_t *ptables;
k_spinlock_key_t key;
int ret;
/*
* For now, lets just assert if we have reached the maximum number
* of asid we assert.
*/
__ASSERT(asid_count < (Z_XTENSA_MMU_SHARED_ASID), "Reached maximum of ASID available");
key = k_spin_lock(&xtensa_mmu_lock);
ptables = dup_table(z_xtensa_kernel_ptables);
if (ptables == NULL) {
ret = -ENOMEM;
goto err;
}
domain->arch.ptables = ptables;
domain->arch.asid = ++asid_count;
sys_slist_append(&xtensa_domain_list, &domain->arch.node);
ret = 0;
err:
k_spin_unlock(&xtensa_mmu_lock, key);
return ret;
}
static int region_map_update(uint32_t *ptables, uintptr_t start,
size_t size, uint32_t ring, uint32_t flags)
{
int ret = 0;
for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) {
uint32_t *l2_table, pte;
uint32_t page = start + offset;
uint32_t l1_pos = page >> 22;
uint32_t l2_pos = Z_XTENSA_L2_POS(page);
/* Make sure we grab a fresh copy of L1 page table */
sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0]));
l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
pte = Z_XTENSA_PTE_RING_SET(l2_table[l2_pos], ring);
pte = Z_XTENSA_PTE_ATTR_SET(pte, flags);
l2_table[l2_pos] = pte;
sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0]));
xtensa_dtlb_vaddr_invalidate(
(void *)(pte & Z_XTENSA_PTE_PPN_MASK));
}
return ret;
}
static inline int update_region(uint32_t *ptables, uintptr_t start,
size_t size, uint32_t ring, uint32_t flags)
{
int ret;
k_spinlock_key_t key;
key = k_spin_lock(&xtensa_mmu_lock);
#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP
uintptr_t va, va_uc;
uint32_t new_flags, new_flags_uc;
if (arch_xtensa_is_ptr_cached((void *)start)) {
va = start;
va_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)start);
} else {
va = (uintptr_t)arch_xtensa_cached_ptr((void *)start);
va_uc = start;
}
new_flags_uc = (flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK);
new_flags = new_flags_uc | Z_XTENSA_MMU_CACHED_WB;
ret = region_map_update(ptables, va, size, ring, new_flags);
if (ret == 0) {
ret = region_map_update(ptables, va_uc, size, ring, new_flags_uc);
}
#else
ret = region_map_update(ptables, start, size, ring, flags);
#endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */
k_spin_unlock(&xtensa_mmu_lock, key);
return ret;
}
static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size)
{
return update_region(ptables, start, size, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_W);
}
void xtensa_set_stack_perms(struct k_thread *thread)
{
if ((thread->base.user_options & K_USER) == 0) {
return;
}
update_region(thread_page_tables_get(thread),
thread->stack_info.start, thread->stack_info.size,
Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB);
}
void xtensa_user_stack_perms(struct k_thread *thread)
{
(void)memset((void *)thread->stack_info.start, 0xAA,
thread->stack_info.size - thread->stack_info.delta);
update_region(thread_page_tables_get(thread),
thread->stack_info.start, thread->stack_info.size,
Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB);
}
int arch_mem_domain_max_partitions_get(void)
{
return CONFIG_MAX_DOMAIN_PARTITIONS;
}
int arch_mem_domain_partition_remove(struct k_mem_domain *domain,
uint32_t partition_id)
{
struct k_mem_partition *partition = &domain->partitions[partition_id];
/* Reset the partition's region back to defaults */
return reset_region(domain->arch.ptables, partition->start,
partition->size);
}
int arch_mem_domain_partition_add(struct k_mem_domain *domain,
uint32_t partition_id)
{
uint32_t ring = domain->arch.asid == 0 ? Z_XTENSA_KERNEL_RING : Z_XTENSA_USER_RING;
struct k_mem_partition *partition = &domain->partitions[partition_id];
return update_region(domain->arch.ptables, partition->start,
partition->size, ring, partition->attr);
}
/* These APIs don't need to do anything */
int arch_mem_domain_thread_add(struct k_thread *thread)
{
int ret = 0;
bool is_user, is_migration;
uint32_t *old_ptables;
struct k_mem_domain *domain;
old_ptables = thread->arch.ptables;
domain = thread->mem_domain_info.mem_domain;
thread->arch.ptables = domain->arch.ptables;
is_user = (thread->base.user_options & K_USER) != 0;
is_migration = (old_ptables != NULL) && is_user;
/* Give access to the thread's stack in its new
* memory domain if it is migrating.
*/
if (is_migration) {
xtensa_set_stack_perms(thread);
}
if (is_migration) {
ret = reset_region(old_ptables,
thread->stack_info.start,
thread->stack_info.size);
}
return ret;
}
int arch_mem_domain_thread_remove(struct k_thread *thread)
{
struct k_mem_domain *domain = thread->mem_domain_info.mem_domain;
if ((thread->base.user_options & K_USER) == 0) {
return 0;
}
if ((thread->base.thread_state & _THREAD_DEAD) == 0) {
/* Thread is migrating to another memory domain and not
* exiting for good; we weren't called from
* z_thread_abort(). Resetting the stack region will
* take place in the forthcoming thread_add() call.
*/
return 0;
}
/* Restore permissions on the thread's stack area since it is no
* longer a member of the domain.
*/
return reset_region(domain->arch.ptables,
thread->stack_info.start,
thread->stack_info.size);
}
static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool write)
{
uint8_t asid_ring;
uint32_t rasid, pte, *l2_table;
uint32_t l1_pos = page >> 22;
uint32_t l2_pos = Z_XTENSA_L2_POS(page);
if (is_pte_illegal(ptables[l1_pos])) {
return false;
}
l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK);
pte = l2_table[l2_pos];
if (is_pte_illegal(pte)) {
return false;
}
asid_ring = 0;
rasid = xtensa_rasid_get();
for (uint32_t i = 0; i < 4; i++) {
if (Z_XTENSA_PTE_ASID_GET(pte, rasid) ==
Z_XTENSA_RASID_ASID_GET(rasid, i)) {
asid_ring = i;
break;
}
}
if (ring > asid_ring) {
return false;
}
if (write) {
return (Z_XTENSA_PTE_ATTR_GET((pte)) & Z_XTENSA_MMU_W) != 0;
}
return true;
}
int arch_buffer_validate(void *addr, size_t size, int write)
{
int ret = 0;
uint8_t *virt;
size_t aligned_size;
const struct k_thread *thread = _current;
uint32_t *ptables = thread_page_tables_get(thread);
uint8_t ring = ((thread->base.user_options & K_USER) != 0) ?
Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING;
/* addr/size arbitrary, fix this up into an aligned region */
k_mem_region_align((uintptr_t *)&virt, &aligned_size,
(uintptr_t)addr, size, CONFIG_MMU_PAGE_SIZE);
for (size_t offset = 0; offset < aligned_size;
offset += CONFIG_MMU_PAGE_SIZE) {
if (!page_validate(ptables, (uint32_t)(virt + offset), ring, write)) {
ret = -1;
break;
}
}
return ret;
}
void z_xtensa_swap_update_page_tables(struct k_thread *incoming)
{
uint32_t *ptables = incoming->arch.ptables;
struct arch_mem_domain *domain =
&(incoming->mem_domain_info.mem_domain->arch);
/* Lets set the asid for the incoming thread */
if ((incoming->base.user_options & K_USER) != 0) {
xtensa_rasid_asid_set(domain->asid, Z_XTENSA_USER_RING);
}
switch_page_tables(ptables, true, false);
}
#endif /* CONFIG_USERSPACE */

View file

@ -174,6 +174,13 @@ static inline bool arch_is_in_isr(void)
return arch_curr_cpu()->nested != 0U;
}
#ifdef CONFIG_USERSPACE
extern void z_xtensa_userspace_enter(k_thread_entry_t user_entry,
void *p1, void *p2, void *p3,
uintptr_t stack_end,
uintptr_t stack_start);
#endif /* CONFIG_USERSPACE */
#ifdef __cplusplus
}
#endif

View file

@ -2,4 +2,18 @@
* Copyright (c) 2021 Intel Corporation
* SPDX-License-Identifier: Apache-2.0
*/
/* Empty File */
#ifndef ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_
#define ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_
#define _thread_offset_to_flags \
(___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET)
#ifdef CONFIG_USERSPACE
#define _thread_offset_to_psp \
(___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET)
#define _thread_offset_to_ptables \
(___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET)
#endif /* CONFIG_USERSPACE */
#endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ */

View file

@ -176,7 +176,8 @@
rsr.SCOMPARE1 a0
s32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET
#endif
#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE)
#if XCHAL_HAVE_THREADPTR && \
(defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE))
rur.THREADPTR a0
s32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET
#endif
@ -409,6 +410,16 @@ _xstack_returned_\@:
l32i a2, a1, 0
l32i a2, a2, ___xtensa_irq_bsa_t_scratch_OFFSET
#if XCHAL_HAVE_THREADPTR && defined(CONFIG_USERSPACE)
/* Clear up the threadptr because it is used
* to check if a thread is runnig on user mode. Since
* we are in a interruption we don't want the system
* thinking it is possbly running in user mode.
*/
movi.n a0, 0
wur.THREADPTR a0
#endif /* XCHAL_HAVE_THREADPTR && CONFIG_USERSPACE */
/* There's a gotcha with level 1 handlers: the INTLEVEL field
* gets left at zero and not set like high priority interrupts
* do. That works fine for exceptions, but for L1 interrupts,

View file

@ -23,6 +23,8 @@
#include <zephyr/arch/arc/syscall.h>
#elif defined(CONFIG_RISCV)
#include <zephyr/arch/riscv/syscall.h>
#elif defined(CONFIG_XTENSA)
#include <zephyr/arch/xtensa/syscall.h>
#endif
#endif /* ZEPHYR_INCLUDE_ARCH_SYSCALL_H_ */

View file

@ -29,6 +29,7 @@
#include <zephyr/arch/common/addr_types.h>
#include <zephyr/arch/xtensa/gdbstub.h>
#include <zephyr/debug/sparse.h>
#include <zephyr/sys/slist.h>
#include <zephyr/arch/xtensa/xtensa_mmu.h>
@ -47,6 +48,15 @@
extern "C" {
#endif
struct arch_mem_domain {
#ifdef CONFIG_XTENSA_MMU
uint32_t *ptables __aligned(CONFIG_MMU_PAGE_SIZE);
uint8_t asid;
bool dirty;
#endif
sys_snode_t node;
};
extern void xtensa_arch_except(int reason_p);
#define ARCH_EXCEPT(reason_p) do { \

View file

@ -0,0 +1,238 @@
/*
* Copyright (c) 2022 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
/**
* @file
* @brief Xtensa specific syscall header
*
* This header contains the Xtensa specific syscall interface. It is
* included by the syscall interface architecture-abstraction header
* (include/arch/syscall.h)
*/
#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_
#define ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_
#ifdef CONFIG_USERSPACE
#ifndef _ASMLANGUAGE
#include <zephyr/types.h>
#include <stdbool.h>
#include <zephyr/linker/sections.h>
#include <zephyr/sys/util_macro.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t arg5, uintptr_t arg6,
uintptr_t call_id);
uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t arg5,
uintptr_t call_id);
uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t call_id);
uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t call_id);
uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2,
uintptr_t call_id);
uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id);
uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id);
#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */
/**
* We are following Linux Xtensa syscall ABI:
*
* syscall number arg1, arg2, arg3, arg4, arg5, arg6
* -------------- ----------------------------------
* a2 a6, a3, a4, a5, a8, a9
*
**/
static inline uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t arg5, uintptr_t arg6,
uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke6_helper(arg1, arg2, arg3,
arg4, arg5, arg6,
call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
register uintptr_t a5 __asm__("%a5") = arg4;
register uintptr_t a8 __asm__("%a8") = arg5;
register uintptr_t a9 __asm__("%a9") = arg6;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4),
"r" (a5), "r" (a8), "r" (a9)
: "memory");
return a2;
#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */
}
static inline uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t arg5, uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke5_helper(arg1, arg2, arg3,
arg4, arg5, call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
register uintptr_t a5 __asm__("%a5") = arg4;
register uintptr_t a8 __asm__("%a8") = arg5;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4),
"r" (a5), "r" (a8)
: "memory");
return a2;
#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */
}
static inline uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t arg4,
uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke4_helper(arg1, arg2, arg3, arg4, call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
register uintptr_t a5 __asm__("%a5") = arg4;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4),
"r" (a5)
: "memory");
return a2;
#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */
}
static inline uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2,
uintptr_t arg3, uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke3_helper(arg1, arg2, arg3, call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
register uintptr_t a4 __asm__("%a4") = arg3;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3), "r" (a4)
: "memory");
return a2;
#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */
}
static inline uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2,
uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke2_helper(arg1, arg2, call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
register uintptr_t a3 __asm__("%a3") = arg2;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6), "r" (a3)
: "memory");
return a2;
#endif
}
static inline uintptr_t arch_syscall_invoke1(uintptr_t arg1,
uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke1_helper(arg1, call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
register uintptr_t a6 __asm__("%a6") = arg1;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2), "r" (a6)
: "memory");
return a2;
#endif
}
static inline uintptr_t arch_syscall_invoke0(uintptr_t call_id)
{
#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER
return arch_syscall_invoke0_helper(call_id);
#else
register uintptr_t a2 __asm__("%a2") = call_id;
__asm__ volatile("syscall\n\t"
: "=r" (a2)
: "r" (a2)
: "memory");
return a2;
#endif
}
/*
* There is no easy (or generic) way to figure out if a thread is runnining
* in un-privileged mode. Reading the currrent ring (PS.CRING) is a privileged
* instruction and not thread local storage is not available in xcc.
*/
static inline bool arch_is_user_context(void)
{
uint32_t thread;
__asm__ volatile(
"rur.THREADPTR %0\n\t"
: "=a" (thread)
);
return !!thread;
}
#ifdef __cplusplus
}
#endif
#endif /* _ASMLANGUAGE */
#endif /* CONFIG_USERSPACE */
#endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ */

View file

@ -7,6 +7,7 @@
#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_
#define ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_
#include <stdint.h>
#ifndef _ASMLANGUAGE
/* Xtensa doesn't use these structs, but Zephyr core requires they be
@ -22,6 +23,14 @@ typedef struct _callee_saved _callee_saved_t;
struct _thread_arch {
uint32_t last_cpu;
#ifdef CONFIG_USERSPACE
uint32_t *ptables;
/* Initial privilege mode stack pointer when doing a system call.
* Un-set for surpervisor threads.
*/
uint8_t *psp;
#endif
};
typedef struct _thread_arch _thread_arch_t;

View file

@ -9,8 +9,40 @@
#define Z_XTENSA_MMU_X BIT(0)
#define Z_XTENSA_MMU_W BIT(1)
#define Z_XTENSA_MMU_XW (BIT(1) | BIT(0))
#define Z_XTENSA_MMU_CACHED_WB BIT(2)
#define Z_XTENSA_MMU_CACHED_WT BIT(3)
#define K_MEM_PARTITION_IS_EXECUTABLE(attr) (((attr) & Z_XTENSA_MMU_X) != 0)
#define K_MEM_PARTITION_IS_WRITABLE(attr) (((attr) & Z_XENSA_MMU_W) != 0)
/* Read-Write access permission attributes */
#define K_MEM_PARTITION_P_RW_U_RW ((k_mem_partition_attr_t) \
{Z_XTENSA_MMU_W})
#define K_MEM_PARTITION_P_RW_U_NA ((k_mem_partition_attr_t) \
{0})
#define K_MEM_PARTITION_P_RO_U_RO ((k_mem_partition_attr_t) \
{0})
#define K_MEM_PARTITION_P_RO_U_NA ((k_mem_partition_attr_t) \
{0})
#define K_MEM_PARTITION_P_NA_U_NA ((k_mem_partition_attr_t) \
{0})
/* Execution-allowed attributes */
#define K_MEM_PARTITION_P_RX_U_RX ((k_mem_partition_attr_t) \
{Z_XTENSA_MMU_X})
/*
* This BIT tells the mapping code whether the uncached pointer should
* be shared between all threads. That is not used in the HW, it is
* just for the implementation.
*
* The pte mapping this memory will use an ASID that is set in the
* ring 4 spot in RASID.
*/
#define Z_XTENSA_MMU_MAP_SHARED BIT(30)
#define Z_XTENSA_MMU_ILLEGAL (BIT(3) | BIT(2))
/* Struct used to map a memory region */
@ -21,6 +53,8 @@ struct xtensa_mmu_range {
const uint32_t attrs;
};
typedef uint32_t k_mem_partition_attr_t;
extern const struct xtensa_mmu_range xtensa_soc_mmu_ranges[];
extern int xtensa_soc_mmu_ranges_num;