michaelh/zephyr
1
0
Fork 0
Code Releases Activity
zephyr/arch/riscv/core/isr.S
Nicolas Pitre 26d7bd47a0 riscv: decouple the Zephyr CPU number from the hart ID 
Currently it is assumed that Zephyr CPU numbers match their hartid
value one for one. This assumption was relied upon to efficiently
retrieve the current CPU's `struct _cpu` pointer.

People are starting to have systems with a mix of different usage for
each CPU and such assumption may no longer be true.

Let's completely decouple the hartid from the Zephyr CPU number by
stuffing each CPU's `struct _cpu` pointer in their respective scratch
register instead. `arch_curr_cpu()` becomes more efficient as well.

Since the scratch register was previously used to store userspace's
exception stack pointer, that is now moved into `struct _cpu_arch`
which implied minor user space entry code cleanup and rationalization.

Signed-off-by: Nicolas Pitre <npitre@baylibre.com>
2023-01-19 13:48:42 +01:00

626 lines
16 KiB
ArmAsm
Raw Blame History

/*
* Copyright (c) 2016 Jean-Paul Etienne <fractalclone@gmail.com>
* Copyright (c) 2018 Foundries.io Ltd
* Copyright (c) 2020 BayLibre, SAS
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/toolchain.h>
#include <zephyr/linker/sections.h>
#include <offsets_short.h>
#include <zephyr/arch/cpu.h>
#include <zephyr/sys/util.h>
#include <zephyr/kernel.h>
#include <zephyr/syscall.h>
#include <zephyr/arch/riscv/csr.h>
#include <zephyr/arch/riscv/syscall.h>
#include "asm_macros.inc"
#ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING
#include <soc_isr_stacking.h>
#endif
/* Convenience macros for loading/storing register states. */
#define DO_FP_CALLER_SAVED(op, reg) \
op ft0, __z_arch_esf_t_ft0_OFFSET(reg) ;\
op ft1, __z_arch_esf_t_ft1_OFFSET(reg) ;\
op ft2, __z_arch_esf_t_ft2_OFFSET(reg) ;\
op ft3, __z_arch_esf_t_ft3_OFFSET(reg) ;\
op ft4, __z_arch_esf_t_ft4_OFFSET(reg) ;\
op ft5, __z_arch_esf_t_ft5_OFFSET(reg) ;\
op ft6, __z_arch_esf_t_ft6_OFFSET(reg) ;\
op ft7, __z_arch_esf_t_ft7_OFFSET(reg) ;\
op ft8, __z_arch_esf_t_ft8_OFFSET(reg) ;\
op ft9, __z_arch_esf_t_ft9_OFFSET(reg) ;\
op ft10, __z_arch_esf_t_ft10_OFFSET(reg) ;\
op ft11, __z_arch_esf_t_ft11_OFFSET(reg) ;\
op fa0, __z_arch_esf_t_fa0_OFFSET(reg) ;\
op fa1, __z_arch_esf_t_fa1_OFFSET(reg) ;\
op fa2, __z_arch_esf_t_fa2_OFFSET(reg) ;\
op fa3, __z_arch_esf_t_fa3_OFFSET(reg) ;\
op fa4, __z_arch_esf_t_fa4_OFFSET(reg) ;\
op fa5, __z_arch_esf_t_fa5_OFFSET(reg) ;\
op fa6, __z_arch_esf_t_fa6_OFFSET(reg) ;\
op fa7, __z_arch_esf_t_fa7_OFFSET(reg) ;
#define DO_CALLER_SAVED(op) \
RV_E( op t0, __z_arch_esf_t_t0_OFFSET(sp) );\
RV_E( op t1, __z_arch_esf_t_t1_OFFSET(sp) );\
RV_E( op t2, __z_arch_esf_t_t2_OFFSET(sp) );\
RV_I( op t3, __z_arch_esf_t_t3_OFFSET(sp) );\
RV_I( op t4, __z_arch_esf_t_t4_OFFSET(sp) );\
RV_I( op t5, __z_arch_esf_t_t5_OFFSET(sp) );\
RV_I( op t6, __z_arch_esf_t_t6_OFFSET(sp) );\
RV_E( op a0, __z_arch_esf_t_a0_OFFSET(sp) );\
RV_E( op a1, __z_arch_esf_t_a1_OFFSET(sp) );\
RV_E( op a2, __z_arch_esf_t_a2_OFFSET(sp) );\
RV_E( op a3, __z_arch_esf_t_a3_OFFSET(sp) );\
RV_E( op a4, __z_arch_esf_t_a4_OFFSET(sp) );\
RV_E( op a5, __z_arch_esf_t_a5_OFFSET(sp) );\
RV_I( op a6, __z_arch_esf_t_a6_OFFSET(sp) );\
RV_I( op a7, __z_arch_esf_t_a7_OFFSET(sp) );\
RV_E( op ra, __z_arch_esf_t_ra_OFFSET(sp) )
.macro get_current_cpu dst
#if defined(CONFIG_SMP) || defined(CONFIG_USERSPACE)
csrr \dst, mscratch
#else
la \dst, _kernel + ___kernel_t_cpus_OFFSET
#endif
.endm
/* imports */
GDATA(_sw_isr_table)
GTEXT(__soc_is_irq)
GTEXT(__soc_handle_irq)
GTEXT(_Fault)
#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE
GTEXT(__soc_save_context)
GTEXT(__soc_restore_context)
#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */
GTEXT(z_riscv_fatal_error)
GTEXT(z_get_next_switch_handle)
GTEXT(z_riscv_switch)
GTEXT(z_riscv_thread_start)
#ifdef CONFIG_TRACING
GTEXT(sys_trace_isr_enter)
GTEXT(sys_trace_isr_exit)
#endif
#ifdef CONFIG_USERSPACE
GDATA(_k_syscall_table)
#endif
/* exports */
GTEXT(_isr_wrapper)
/* use ABI name of registers for the sake of simplicity */
/*
* Generic architecture-level IRQ handling, along with callouts to
* SoC-specific routines.
*
* Architecture level IRQ handling includes basic context save/restore
* of standard registers and calling ISRs registered at Zephyr's driver
* level.
*
* Since RISC-V does not completely prescribe IRQ handling behavior,
* implementations vary (some implementations also deviate from
* what standard behavior is defined). Hence, the arch level code expects
* the following functions to be provided at the SOC level:
*
* - __soc_is_irq: decide if we're handling an interrupt or an exception
* - __soc_handle_irq: handle SoC-specific details for a pending IRQ
* (e.g. clear a pending bit in a SoC-specific register)
*
* If CONFIG_RISCV_SOC_CONTEXT_SAVE=y, calls to SoC-level context save/restore
* routines are also made here. For details, see the Kconfig help text.
*/
/*
* Handler called upon each exception/interrupt/fault
*/
SECTION_FUNC(exception.entry, _isr_wrapper)
#ifdef CONFIG_USERSPACE
/* retrieve address of _current_cpu preserving s0 */
csrrw s0, mscratch, s0
/* preserve t0 and t1 temporarily */
sr t0, _curr_cpu_arch_user_exc_tmp0(s0)
sr t1, _curr_cpu_arch_user_exc_tmp1(s0)
/* determine if we come from user space */
csrr t0, mstatus
li t1, MSTATUS_MPP
and t0, t0, t1
bnez t0, 1f
/* in user space we were: switch to our privileged stack */
mv t0, sp
lr sp, _curr_cpu_arch_user_exc_sp(s0)
/* Save user stack value. Coming from user space, we know this
* can't overflow the privileged stack. The esf will be allocated
* later but it is safe to store our saved user sp here. */
sr t0, (-__z_arch_esf_t_SIZEOF + __z_arch_esf_t_sp_OFFSET)(sp)
/* Make sure tls pointer is sane */
lr t0, ___cpu_t_current_OFFSET(s0)
lr tp, _thread_offset_to_tls(t0)
/* Clear our per-thread usermode flag */
lui t0, %tprel_hi(is_user_mode)
add t0, t0, tp, %tprel_add(is_user_mode)
sb zero, %tprel_lo(is_user_mode)(t0)
1:
/* retrieve original t0/t1 values */
lr t0, _curr_cpu_arch_user_exc_tmp0(s0)
lr t1, _curr_cpu_arch_user_exc_tmp1(s0)
/* retrieve original s0 and restore _current_cpu in mscratch */
csrrw s0, mscratch, s0
#endif
#ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING
SOC_ISR_SW_STACKING
#else
/* Save caller-saved registers on current thread stack. */
addi sp, sp, -__z_arch_esf_t_SIZEOF
DO_CALLER_SAVED(sr) ;
#endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */
/* Save s0 in the esf and load it with &_current_cpu. */
sr s0, __z_arch_esf_t_s0_OFFSET(sp)
get_current_cpu s0
/* Save MEPC register */
csrr t0, mepc
sr t0, __z_arch_esf_t_mepc_OFFSET(sp)
/* Save MSTATUS register */
csrr t2, mstatus
sr t2, __z_arch_esf_t_mstatus_OFFSET(sp)
#if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING)
/* Assess whether floating-point registers need to be saved. */
li t1, MSTATUS_FS_INIT
and t0, t2, t1
beqz t0, skip_store_fp_caller_saved
DO_FP_CALLER_SAVED(fsr, sp)
skip_store_fp_caller_saved:
#endif /* CONFIG_FPU && CONFIG_FPU_SHARING */
#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE
/* Handle context saving at SOC level. */
addi a0, sp, __z_arch_esf_t_soc_context_OFFSET
jal ra, __soc_save_context
#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */
/*
* Check if exception is the result of an interrupt or not.
* (SOC dependent). Following the RISC-V architecture spec, the MSB
* of the mcause register is used to indicate whether an exception
* is the result of an interrupt or an exception/fault. But for some
* SOCs (like pulpino or riscv-qemu), the MSB is never set to indicate
* interrupt. Hence, check for interrupt/exception via the __soc_is_irq
* function (that needs to be implemented by each SOC). The result is
* returned via register a0 (1: interrupt, 0 exception)
*/
jal ra, __soc_is_irq
/* If a0 != 0, jump to is_interrupt */
bnez a0, is_interrupt
/*
* If the exception is the result of an ECALL, check whether to
* perform a context-switch or an IRQ offload. Otherwise call _Fault
* to report the exception.
*/
csrr t0, mcause
li t2, SOC_MCAUSE_EXP_MASK
and t0, t0, t2
/*
* If mcause == SOC_MCAUSE_ECALL_EXP, handle system call from
* kernel thread.
*/
li t1, SOC_MCAUSE_ECALL_EXP
beq t0, t1, is_kernel_syscall
#ifdef CONFIG_USERSPACE
/*
* If mcause == SOC_MCAUSE_USER_ECALL_EXP, handle system call
* for user mode thread.
*/
li t1, SOC_MCAUSE_USER_ECALL_EXP
beq t0, t1, is_user_syscall
#endif /* CONFIG_USERSPACE */
/*
* Call _Fault to handle exception.
* Stack pointer is pointing to a z_arch_esf_t structure, pass it
* to _Fault (via register a0).
* If _Fault shall return, set return address to
* no_reschedule to restore stack.
*/
mv a0, sp
la ra, no_reschedule
tail _Fault
is_kernel_syscall:
/*
* A syscall is the result of an ecall instruction, in which case the
* MEPC will contain the address of the ecall instruction.
* Increment saved MEPC by 4 to prevent triggering the same ecall
* again upon exiting the ISR.
*
* It's safe to always increment by 4, even with compressed
* instructions, because the ecall instruction is always 4 bytes.
*/
lr t0, __z_arch_esf_t_mepc_OFFSET(sp)
addi t0, t0, 4
sr t0, __z_arch_esf_t_mepc_OFFSET(sp)
#ifdef CONFIG_PMP_STACK_GUARD
/* Re-activate PMP for m-mode */
li t1, MSTATUS_MPP
csrc mstatus, t1
li t1, MSTATUS_MPRV
csrs mstatus, t1
#endif
/* Determine what to do. Operation code is in t0. */
lr t0, __z_arch_esf_t_t0_OFFSET(sp)
.if RV_ECALL_RUNTIME_EXCEPT != 0; .err; .endif
beqz t0, do_fault
#if defined(CONFIG_IRQ_OFFLOAD)
li t1, RV_ECALL_IRQ_OFFLOAD
beq t0, t1, do_irq_offload
#endif
#ifdef CONFIG_RISCV_ALWAYS_SWITCH_THROUGH_ECALL
li t1, RV_ECALL_SCHEDULE
bne t0, t1, skip_schedule
lr a0, __z_arch_esf_t_a0_OFFSET(sp)
lr a1, __z_arch_esf_t_a1_OFFSET(sp)
j reschedule
skip_schedule:
#endif
/* default fault code is K_ERR_KERNEL_OOPS */
li a0, 3
j 1f
do_fault:
/* Handle RV_ECALL_RUNTIME_EXCEPT. Retrieve reason in a0, esf in A1. */
lr a0, __z_arch_esf_t_a0_OFFSET(sp)
1: mv a1, sp
tail z_riscv_fatal_error
#if defined(CONFIG_IRQ_OFFLOAD)
do_irq_offload:
/*
* Retrieve provided routine and argument from the stack.
* Routine pointer is in saved a0, argument in saved a1
* so we load them with a1/a0 (reversed).
*/
lr a1, __z_arch_esf_t_a0_OFFSET(sp)
lr a0, __z_arch_esf_t_a1_OFFSET(sp)
/* Increment _current_cpu->nested */
lw t1, ___cpu_t_nested_OFFSET(s0)
addi t2, t1, 1
sw t2, ___cpu_t_nested_OFFSET(s0)
bnez t1, 1f
/* Switch to interrupt stack */
mv t0, sp
lr sp, ___cpu_t_irq_stack_OFFSET(s0)
/* Save thread stack pointer on interrupt stack */
addi sp, sp, -16
sr t0, 0(sp)
1:
/* Execute provided routine (argument is in a0 already). */
jalr ra, a1, 0
/* Leave through the regular IRQ exit path */
j irq_done
#endif /* CONFIG_IRQ_OFFLOAD */
#ifdef CONFIG_USERSPACE
is_user_syscall:
#ifdef CONFIG_PMP_STACK_GUARD
/*
* We came from userspace and need to reconfigure the
* PMP for kernel mode stack guard.
*/
lr a0, ___cpu_t_current_OFFSET(s0)
call z_riscv_pmp_stackguard_enable
#endif
/* It is safe to re-enable IRQs now */
csrs mstatus, MSTATUS_IEN
/*
* Same as for is_kernel_syscall: increment saved MEPC by 4 to
* prevent triggering the same ecall again upon exiting the ISR.
*/
lr t1, __z_arch_esf_t_mepc_OFFSET(sp)
addi t1, t1, 4
sr t1, __z_arch_esf_t_mepc_OFFSET(sp)
/* Restore argument registers from user stack */
lr a0, __z_arch_esf_t_a0_OFFSET(sp)
lr a1, __z_arch_esf_t_a1_OFFSET(sp)
lr a2, __z_arch_esf_t_a2_OFFSET(sp)
lr a3, __z_arch_esf_t_a3_OFFSET(sp)
lr a4, __z_arch_esf_t_a4_OFFSET(sp)
lr a5, __z_arch_esf_t_a5_OFFSET(sp)
lr t0, __z_arch_esf_t_t0_OFFSET(sp)
#if defined(CONFIG_RISCV_ISA_RV32E)
/* Stack alignment for RV32E is 4 bytes */
addi sp, sp, -4
mv t1, sp
sw t1, 0(sp)
#else
mv a6, sp
#endif /* CONFIG_RISCV_ISA_RV32E */
/* validate syscall limit */
li t1, K_SYSCALL_LIMIT
bltu t0, t1, valid_syscall_id
/* bad syscall id. Set arg1 to bad id and set call_id to SYSCALL_BAD */
mv a0, t0
li t0, K_SYSCALL_BAD
valid_syscall_id:
la t2, _k_syscall_table
slli t1, t0, RV_REGSHIFT # Determine offset from indice value
add t2, t2, t1 # Table addr + offset = function addr
lr t2, 0(t2) # Load function address
/* Execute syscall function */
jalr ra, t2, 0
#if defined(CONFIG_RISCV_ISA_RV32E)
addi sp, sp, 4
#endif /* CONFIG_RISCV_ISA_RV32E */
/* Update a0 (return value) on the stack */
sr a0, __z_arch_esf_t_a0_OFFSET(sp)
/* Disable IRQs again before leaving */
csrc mstatus, MSTATUS_IEN
j might_have_rescheduled
#endif /* CONFIG_USERSPACE */
is_interrupt:
#ifdef CONFIG_PMP_STACK_GUARD
#ifdef CONFIG_USERSPACE
/*
* If we came from userspace then we need to reconfigure the
* PMP for kernel mode stack guard.
*/
lr t0, __z_arch_esf_t_mstatus_OFFSET(sp)
li t1, MSTATUS_MPP
and t0, t0, t1
bnez t0, 1f
lr a0, ___cpu_t_current_OFFSET(s0)
call z_riscv_pmp_stackguard_enable
j 2f
#endif /* CONFIG_USERSPACE */
1: /* Re-activate PMP for m-mode */
li t1, MSTATUS_MPP
csrc mstatus, t1
li t1, MSTATUS_MPRV
csrs mstatus, t1
2:
#endif
/* Increment _current_cpu->nested */
lw t1, ___cpu_t_nested_OFFSET(s0)
addi t2, t1, 1
sw t2, ___cpu_t_nested_OFFSET(s0)
bnez t1, on_irq_stack
/* Switch to interrupt stack */
mv t0, sp
lr sp, ___cpu_t_irq_stack_OFFSET(s0)
/*
* Save thread stack pointer on interrupt stack
* In RISC-V, stack pointer needs to be 16-byte aligned
*/
addi sp, sp, -16
sr t0, 0(sp)
on_irq_stack:
#ifdef CONFIG_TRACING_ISR
call sys_trace_isr_enter
#endif
/* Get IRQ causing interrupt */
csrr a0, mcause
li t0, SOC_MCAUSE_EXP_MASK
and a0, a0, t0
/*
* Clear pending IRQ generating the interrupt at SOC level
* Pass IRQ number to __soc_handle_irq via register a0
*/
jal ra, __soc_handle_irq
/*
* Call corresponding registered function in _sw_isr_table.
* (table is 2-word wide, we should shift index accordingly)
*/
la t0, _sw_isr_table
slli a0, a0, (RV_REGSHIFT + 1)
add t0, t0, a0
/* Load argument in a0 register */
lr a0, 0(t0)
/* Load ISR function address in register t1 */
lr t1, RV_REGSIZE(t0)
/* Call ISR function */
jalr ra, t1, 0
#ifdef CONFIG_TRACING_ISR
call sys_trace_isr_exit
#endif
irq_done:
/* Decrement _current_cpu->nested */
lw t2, ___cpu_t_nested_OFFSET(s0)
addi t2, t2, -1
sw t2, ___cpu_t_nested_OFFSET(s0)
bnez t2, no_reschedule
/* nested count is back to 0: Return to thread stack */
lr sp, 0(sp)
#ifdef CONFIG_STACK_SENTINEL
call z_check_stack_sentinel
#endif
check_reschedule:
/* Get pointer to current thread on this CPU */
lr a1, ___cpu_t_current_OFFSET(s0)
/*
* Get next thread to schedule with z_get_next_switch_handle().
* We pass it a NULL as we didn't save the whole thread context yet.
* If no scheduling is necessary then NULL will be returned.
*/
addi sp, sp, -16
sr a1, 0(sp)
mv a0, zero
call z_get_next_switch_handle
lr a1, 0(sp)
addi sp, sp, 16
beqz a0, no_reschedule
reschedule:
/*
* Perform context switch:
* a0 = new thread
* a1 = old thread
*/
call z_riscv_switch
z_riscv_thread_start:
might_have_rescheduled:
#ifdef CONFIG_SMP
/* reload s0 with &_current_cpu as it might have changed */
get_current_cpu s0
#endif
no_reschedule:
#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE
/* Restore context at SOC level */
addi a0, sp, __z_arch_esf_t_soc_context_OFFSET
jal ra, __soc_restore_context
#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */
/* Restore MEPC register */
lr t0, __z_arch_esf_t_mepc_OFFSET(sp)
csrw mepc, t0
/* Restore MSTATUS register */
lr t2, __z_arch_esf_t_mstatus_OFFSET(sp)
csrrw t0, mstatus, t2
#if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING)
/*
* Determine if we need to restore FP regs based on the previous
* (before the csr above) mstatus value available in t0.
*/
li t1, MSTATUS_FS_INIT
and t0, t0, t1
beqz t0, no_fp
/* make sure FP is enabled in the restored mstatus */
csrs mstatus, t1
DO_FP_CALLER_SAVED(flr, sp)
j 1f
no_fp: /* make sure this is reflected in the restored mstatus */
csrc mstatus, t1
1:
#endif /* CONFIG_FPU && CONFIG_FPU_SHARING */
#ifdef CONFIG_USERSPACE
/*
* Check if we are returning to user mode. If so then we must
* set is_user_mode to true and preserve our kernel mode stack for
* the next exception to come.
*/
li t1, MSTATUS_MPP
and t0, t2, t1
bnez t0, 1f
#ifdef CONFIG_PMP_STACK_GUARD
/* Remove kernel stack guard and Reconfigure PMP for user mode */
lr a0, ___cpu_t_current_OFFSET(s0)
call z_riscv_pmp_usermode_enable
#endif
/* Set our per-thread usermode flag */
li t1, 1
lui t0, %tprel_hi(is_user_mode)
add t0, t0, tp, %tprel_add(is_user_mode)
sb t1, %tprel_lo(is_user_mode)(t0)
/* preserve stack pointer for next exception entry */
add t0, sp, __z_arch_esf_t_SIZEOF
sr t0, _curr_cpu_arch_user_exc_sp(s0)
j 2f
1:
/*
* We are returning to kernel mode. Store the stack pointer to
* be re-loaded further down.
*/
addi t0, sp, __z_arch_esf_t_SIZEOF
sr t0, __z_arch_esf_t_sp_OFFSET(sp)
2:
#endif
/* Restore s0 (it is no longer ours) */
lr s0, __z_arch_esf_t_s0_OFFSET(sp)
#ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING
SOC_ISR_SW_UNSTACKING
#else
/* Restore caller-saved registers from thread stack */
DO_CALLER_SAVED(lr)
#endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */
#ifdef CONFIG_USERSPACE
/* retrieve saved stack pointer */
lr sp, __z_arch_esf_t_sp_OFFSET(sp)
#else
/* remove esf from the stack */
addi sp, sp, __z_arch_esf_t_SIZEOF
#endif
mret
View git blame Copy permalink