/* * Copyright (c) 2017, Intel Corporation * * SPDX-License-Identifier: Apache-2.0 */ #ifndef ZEPHYR_ARCH_XTENSA_INCLUDE_XTENSA_ASM2_S_H #define ZEPHYR_ARCH_XTENSA_INCLUDE_XTENSA_ASM2_S_H #include #include "xtensa_asm2_context.h" #include #include /* Assembler header! This file contains macros designed to be included * only by the assembler. */ #if defined(CONFIG_XTENSA_HIFI_SHARING) .extern _xtensa_hifi_save #endif /* * SPILL_ALL_WINDOWS * * Spills all windowed registers (i.e. registers not visible as * A0-A15) to their ABI-defined spill regions on the stack. * * Unlike the Xtensa HAL implementation, this code requires that the * EXCM and WOE bit be enabled in PS, and relies on repeated hardware * exception handling to do the register spills. The trick is to do a * noop write to the high registers, which the hardware will trap * (into an overflow exception) in the case where those registers are * already used by an existing call frame. Then it rotates the window * and repeats until all but the A0-A3 registers of the original frame * are guaranteed to be spilled, eventually rotating back around into * the original frame. Advantages: * * - Vastly smaller code size * * - More easily maintained if changes are needed to window over/underflow * exception handling. * * - Requires no scratch registers to do its work, so can be used safely in any * context. * * - If the WOE bit is not enabled (for example, in code written for * the CALL0 ABI), this becomes a silent noop and operates compatibly. * * - In memory protection situations, this relies on the existing * exception handlers (and thus their use of the L/S32E * instructions) to execute stores in the protected space. AFAICT, * the HAL routine does not handle this situation and isn't safe: it * will happily write through the "stack pointers" found in * registers regardless of where they might point. * * - Hilariously it's ACTUALLY FASTER than the HAL routine. And not * just a little bit, it's MUCH faster. With a mostly full register * file on an LX6 core (ESP-32) I'm measuring 145 cycles to spill * registers with this vs. 279 (!) to do it with * xthal_spill_windows(). Apparently Xtensa exception handling is * really fast, and no one told their software people. * * Note that as with the Xtensa HAL spill routine, and unlike context * switching code on most sane architectures, the intermediate states * here will have an invalid stack pointer. That means that this code * must not be preempted in any context (i.e. all Zephyr situations) * where the interrupt code will need to use the stack to save the * context. But unlike the HAL, which runs with exceptions masked via * EXCM, this will not: hit needs the overflow handlers unmasked. Use * INTLEVEL instead (which, happily, is what Zephyr's locking does * anyway). */ .macro SPILL_ALL_WINDOWS #if XCHAL_NUM_AREGS == 64 and a12, a12, a12 rotw 3 and a12, a12, a12 rotw 3 and a12, a12, a12 rotw 3 and a12, a12, a12 rotw 3 and a12, a12, a12 rotw 4 #elif XCHAL_NUM_AREGS == 32 and a12, a12, a12 rotw 3 and a12, a12, a12 rotw 3 and a4, a4, a4 rotw 2 #else #error Unrecognized XCHAL_NUM_AREGS #endif .endm #if XCHAL_HAVE_FP && defined(CONFIG_CPU_HAS_FPU) && defined(CONFIG_FPU_SHARING) /* * FPU_REG_SAVE * * Saves the Float Point Unit context registers in the base save * area pointed to by the current stack pointer A1. The Floating-Point * Coprocessor Option adds the FR register file and two User Registers * called FCR and FSR.The FR register file consists of 16 registers of * 32 bits each and is used for all data computation. */ .macro FPU_REG_SAVE rur.fcr a0 s32i a0, a1, ___xtensa_irq_bsa_t_fcr_OFFSET rur.fsr a0 s32i a0, a1, ___xtensa_irq_bsa_t_fsr_OFFSET ssi f0, a1, ___xtensa_irq_bsa_t_fpu0_OFFSET ssi f1, a1, ___xtensa_irq_bsa_t_fpu1_OFFSET ssi f2, a1, ___xtensa_irq_bsa_t_fpu2_OFFSET ssi f3, a1, ___xtensa_irq_bsa_t_fpu3_OFFSET ssi f4, a1, ___xtensa_irq_bsa_t_fpu4_OFFSET ssi f5, a1, ___xtensa_irq_bsa_t_fpu5_OFFSET ssi f6, a1, ___xtensa_irq_bsa_t_fpu6_OFFSET ssi f7, a1, ___xtensa_irq_bsa_t_fpu7_OFFSET ssi f8, a1, ___xtensa_irq_bsa_t_fpu8_OFFSET ssi f9, a1, ___xtensa_irq_bsa_t_fpu9_OFFSET ssi f10, a1, ___xtensa_irq_bsa_t_fpu10_OFFSET ssi f11, a1, ___xtensa_irq_bsa_t_fpu11_OFFSET ssi f12, a1, ___xtensa_irq_bsa_t_fpu12_OFFSET ssi f13, a1, ___xtensa_irq_bsa_t_fpu13_OFFSET ssi f14, a1, ___xtensa_irq_bsa_t_fpu14_OFFSET ssi f15, a1, ___xtensa_irq_bsa_t_fpu15_OFFSET .endm .macro FPU_REG_RESTORE l32i.n a0, a1, ___xtensa_irq_bsa_t_fcr_OFFSET wur.fcr a0 l32i.n a0, a1, ___xtensa_irq_bsa_t_fsr_OFFSET wur.fsr a0 lsi f0, a1, ___xtensa_irq_bsa_t_fpu0_OFFSET lsi f1, a1, ___xtensa_irq_bsa_t_fpu1_OFFSET lsi f2, a1, ___xtensa_irq_bsa_t_fpu2_OFFSET lsi f3, a1, ___xtensa_irq_bsa_t_fpu3_OFFSET lsi f4, a1, ___xtensa_irq_bsa_t_fpu4_OFFSET lsi f5, a1, ___xtensa_irq_bsa_t_fpu5_OFFSET lsi f6, a1, ___xtensa_irq_bsa_t_fpu6_OFFSET lsi f7, a1, ___xtensa_irq_bsa_t_fpu7_OFFSET lsi f8, a1, ___xtensa_irq_bsa_t_fpu8_OFFSET lsi f9, a1, ___xtensa_irq_bsa_t_fpu9_OFFSET lsi f10, a1, ___xtensa_irq_bsa_t_fpu10_OFFSET lsi f11, a1, ___xtensa_irq_bsa_t_fpu11_OFFSET lsi f12, a1, ___xtensa_irq_bsa_t_fpu12_OFFSET lsi f13, a1, ___xtensa_irq_bsa_t_fpu13_OFFSET lsi f14, a1, ___xtensa_irq_bsa_t_fpu14_OFFSET lsi f15, a1, ___xtensa_irq_bsa_t_fpu15_OFFSET .endm #endif /* * ODD_REG_SAVE * * Stashes the oddball shift/loop context registers in the base save * area pointed to by the register specified by parameter BSA_PTR. * On exit, the scratch register specified by parameter SCRATCH_REG * will have been modified, and the shift/loop instructions can be * used freely (though note loops don't work in exceptions for other * reasons!). * * Does not populate or modify the PS/PC save locations. */ .macro ODD_REG_SAVE SCRATCH_REG, BSA_PTR rsr.sar \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_sar_OFFSET #if XCHAL_HAVE_LOOPS rsr.lbeg \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_lbeg_OFFSET rsr.lend \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_lend_OFFSET rsr.lcount \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_lcount_OFFSET #endif rsr.exccause \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_exccause_OFFSET #if XCHAL_HAVE_S32C1I rsr.scompare1 \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_scompare1_OFFSET #endif #if XCHAL_HAVE_THREADPTR && \ (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) rur.THREADPTR \SCRATCH_REG s32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_threadptr_OFFSET #endif .endm /* * ODD_REG_RESTORE * * Restores the oddball shift/loop context registers in the base save * area pointed to by the register specified by parameter BSA_PTR. * On exit, the scratch register specified by parameter SCRATCH_REG * will have been modified. * * Does not restore the PS/PC save locations. */ .macro ODD_REG_RESTORE SCRATCH_REG, BSA_PTR l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_sar_OFFSET wsr.sar \SCRATCH_REG #if XCHAL_HAVE_LOOPS l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_lbeg_OFFSET wsr.lbeg \SCRATCH_REG l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_lend_OFFSET wsr.lend \SCRATCH_REG l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_lcount_OFFSET wsr.lcount \SCRATCH_REG #endif l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_exccause_OFFSET wsr.exccause \SCRATCH_REG #if XCHAL_HAVE_S32C1I l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_scompare1_OFFSET wsr.scompare1 \SCRATCH_REG #endif #if XCHAL_HAVE_THREADPTR && \ (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) l32i \SCRATCH_REG, \BSA_PTR, ___xtensa_irq_bsa_t_threadptr_OFFSET wur.THREADPTR \SCRATCH_REG #endif .endm #if defined(CONFIG_XTENSA_MMU) && defined(CONFIG_USERSPACE) /* * SWAP_PAGE_TABLE * * This swaps the page tables by using the pre-computed register values * inside the architecture-specific memory domain struct. * * THREAD_PTR_REG is input containing pointer to the incoming thread struct. * SC1_REG and SC2_REG are scratch registers. * * Note that all THREAD_PTR_REG, SC1_REG and SC2_REG are all clobbered. * Restore the thread pointer after this if necessary. */ .macro SWAP_PAGE_TABLE THREAD_PTR_REG, SC1_REG, SC2_REG l32i \THREAD_PTR_REG, \THREAD_PTR_REG, _thread_offset_to_mem_domain j _swap_page_table_\@ .align 16 _swap_page_table_\@: l32i \SC1_REG, \THREAD_PTR_REG, _k_mem_domain_offset_to_arch_reg_ptevaddr l32i \SC2_REG, \THREAD_PTR_REG, _k_mem_domain_offset_to_arch_reg_asid wsr \SC1_REG, PTEVADDR wsr \SC2_REG, RASID l32i \SC1_REG, \THREAD_PTR_REG, _k_mem_domain_offset_to_arch_reg_ptepin_as l32i \SC2_REG, \THREAD_PTR_REG, _k_mem_domain_offset_to_arch_reg_ptepin_at wdtlb \SC2_REG, \SC1_REG l32i \SC1_REG, \THREAD_PTR_REG, _k_mem_domain_offset_to_arch_reg_vecpin_as l32i \SC2_REG, \THREAD_PTR_REG, _k_mem_domain_offset_to_arch_reg_vecpin_at wdtlb \SC2_REG, \SC1_REG isync .endm #endif /* CONFIG_XTENSA_MMU && CONFIG_USERSPACE */ /* * CROSS_STACK_CALL * * Sets the stack up carefully such that a "cross stack" call can spill * correctly, then invokes an immediate handler. Note that: * * 0. When spilling a frame, functions find their callEE's stack pointer * (to save A0-A3) from registers. But they find their * already-spilled callER's stack pointer (to save higher GPRs) from * their own stack memory. * * 1. The function that was interrupted ("interruptee") does not need to * be spilled, because it already has been as part of the context * save. So it doesn't need registers allocated for it anywhere. * * 2. Interruptee's caller needs to spill into the space below the * interrupted stack frame, which means that the A1 register it finds * below it needs to contain the old/interrupted stack and not the * context saved one. * * 3. The ISR dispatcher (called "underneath" interruptee) needs to spill * high registers into the space immediately above its own stack frame, * so it needs to find a caller with the "new" stack pointer instead. * * We make this work by inserting TWO 4-register frames between * "interruptee's caller" and "ISR dispatcher". The top one (which * occupies the slot formerly held by "interruptee", whose registers * were saved via external means) holds the "interrupted A1" and the * bottom has the "top of the interrupt stack" which can be either the * word above a new memory area (when handling an interrupt from user * mode) OR the existing "post-context-save" stack pointer (when * handling a nested interrupt). The code works either way. Because * these are both only 4-registers, neither needs its own caller for * spilling. * * The net cost is 32 wasted bytes on the interrupt stack frame to * spill our two "phantom frames" (actually not quite, as we'd need a * few of those words used somewhere for tracking the stack pointers * anyway). But the benefit is that NO REGISTER FRAMES NEED TO BE * SPILLED on interrupt entry. And if we return back into the same * context we interrupted (a common case) no windows need to be * explicitly spilled at all. And in fact in the case where the ISR * uses significant depth on its own stack, the interrupted frames * will be spilled naturally as a standard cost of a function call, * giving register windows something like "zero cost interrupts". * * FIXME: a terrible awful really nifty idea to fix the stack waste * problem would be to use a SINGLE frame between the two stacks, * pre-spill it with one stack pointer for the "lower" call to see and * leave the register SP in place for the "upper" frame to use. * Would require modifying the Window{Over|Under}flow4 exceptions to * know not to spill/fill these special frames, but that's not too * hard, maybe... * * Enter this macro with a valid "context saved" pointer (i.e. SP * should point to a stored pointer which points to one BSA below the * interrupted/old stack) in A1, a handler function in A2, and a "new" * stack pointer (i.e. a pointer to the word ABOVE the allocated stack * area) in A3. Exceptions should be enabled via PS.EXCM, but * PS.INTLEVEL must (!) be set such that no nested interrupts can * arrive (we restore the natural INTLEVEL from the value in ZSR_EPS * just before entering the call). On return A0/1 will be unchanged, * A2 has the return value of the called function, and A3 is * clobbered. A4-A15 become part of called frames and MUST NOT BE IN * USE by the code that expands this macro. The called function gets * the context save handle in A1 as it's first argument. */ .macro CROSS_STACK_CALL /* Since accessing A4-A11 may trigger window overflows so * we need to setup A0 and A1 correctly before putting * the function arguments for the next two callx4 into * A6, A10 and A11. So stach the "context handle" into * ZSR_EPC, which is usable for now similar to ZSR_EPS. */ wsr.ZSR_EPC a1 rsync /* Recover the interrupted SP from the BSA */ l32i a1, a1, 0 l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET addi a1, a1, ___xtensa_irq_bsa_t_SIZEOF mov a6, a3 /* place "new sp" in the next frame's A2 */ rsr.ZSR_EPC a3 /* restore saved "context handle" in A3 */ mov a10, a3 /* pass "context handle" in 2nd frame's A2 */ mov a11, a2 /* handler in 2nd frame's A3, next frame's A7 */ call4 _xstack_call0_\@ mov a1, a3 /* restore original SP */ mov a2, a6 /* copy return value */ j _xstack_returned_\@ .align 4 _xstack_call0_\@: /* We want an ENTRY to set a bit in windowstart and do the * rotation, but we want our own SP. After that, we are * running in a valid frame, so re-enable interrupts. */ entry a1, 16 mov a1, a2 rsr.ZSR_EPS a2 wsr.ps a2 #ifdef CONFIG_USERSPACE /* Save "context handle" in A3 as we need it to determine * if we need to swap page table later. */ mov a3, a6 #endif callx4 a7 /* call handler */ mov a2, a6 /* copy return value */ #ifdef CONFIG_USERSPACE rsil a6, XCHAL_NUM_INTLEVELS /* If "next" handle to be restored is the same as * the current handle, there is no need to swap page * tables or MPU entries since we will return to * the same thread that was interrupted. */ beq a2, a3, _xstack_skip_table_swap_\@ /* Need to switch page tables because the "next" handle * returned above is not the same handle as we started * with. This means we are being restored to another * thread. */ rsr a6, ZSR_CPU l32i a6, a6, ___cpu_t_current_OFFSET #ifdef CONFIG_XTENSA_MMU #ifdef CONFIG_XTENSA_MMU_FLUSH_AUTOREFILL_DTLBS_ON_SWAP call4 xtensa_swap_update_page_tables #else SWAP_PAGE_TABLE a6, a3, a7 #endif #endif #ifdef CONFIG_XTENSA_MPU call4 xtensa_mpu_map_write #endif _xstack_skip_table_swap_\@: #endif /* CONFIG_USERSPACE */ retw _xstack_returned_\@: .endm /* Entry setup for all exceptions and interrupts. Arrive here with * the stack pointer decremented across a base save area, A0-A3 and * PS/PC already spilled to the stack in the BSA, and A2 containing a * level-specific C handler function. * * This is a macro (to allow for unit testing) that expands to a * handler body to which the vectors can jump. It takes two static * (!) arguments: a special register name (which should be set up to * point to some kind of per-CPU record struct) and offsets within * that struct which contains an interrupt stack top and a "nest * count" word. */ .macro EXCINT_HANDLER NEST_OFF, INTSTACK_OFF /* A2 contains our handler function which will get clobbered * by the save. Stash it into the unused "a1" slot in the * BSA and recover it immediately after. Kind of a hack. */ s32i a2, a1, ___xtensa_irq_bsa_t_scratch_OFFSET ODD_REG_SAVE a0, a1 #if XCHAL_HAVE_FP && defined(CONFIG_CPU_HAS_FPU) && defined(CONFIG_FPU_SHARING) FPU_REG_SAVE #endif #if defined(CONFIG_XTENSA_HIFI_SHARING) call0 _xtensa_hifi_save /* Save HiFi registers */ #endif call0 xtensa_save_high_regs l32i a2, a1, 0 l32i a2, a2, ___xtensa_irq_bsa_t_scratch_OFFSET #if XCHAL_HAVE_THREADPTR && defined(CONFIG_USERSPACE) /* Clear up the threadptr because it is used * to check if a thread is runnig on user mode. Since * we are in a interruption we don't want the system * thinking it is possbly running in user mode. */ movi.n a0, 0 wur.THREADPTR a0 #endif /* XCHAL_HAVE_THREADPTR && CONFIG_USERSPACE */ /* Setting up the cross stack call below has states where the * resulting frames are invalid/non-reentrant, so we can't * allow nested interrupts. But we do need EXCM unmasked, as * we use CALL/ENTRY instructions in the process and need to * handle exceptions to spill caller/interruptee frames. Use * PS.INTLEVEL at maximum to mask all interrupts and stash the * current value in our designated EPS register (which is * guaranteed unused across the call) */ rsil a0, 0xf /* Since we are unmasking EXCM, we need to set RING bits to kernel * mode, otherwise we won't be able to run the exception handler in C. */ movi a3, ~(PS_EXCM_MASK) & ~(PS_RING_MASK) and a0, a0, a3 #ifdef CONFIG_XTENSA_INTERRUPT_NONPREEMPTABLE /* Setting the interrupt mask to the max non-debug level * to prevent lower priority interrupts being preempted by * high level interrupts until processing of that lower level * interrupt has completed. */ movi a3, ~(PS_INTLEVEL_MASK) and a0, a0, a3 movi a3, PS_INTLEVEL(ZSR_RFI_LEVEL) or a0, a0, a3 wsr.ZSR_EPS a0 #else /* There's a gotcha with level 1 handlers: the INTLEVEL field * gets left at zero and not set like high priority interrupts * do. That works fine for exceptions, but for L1 interrupts, * when we unmask EXCM below, the CPU will just fire the * interrupt again and get stuck in a loop blasting save * frames down the stack to the bottom of memory. It would be * good to put this code into the L1 handler only, but there's * not enough room in the vector without some work there to * squash it some. Next choice would be to make this a macro * argument and expand two versions of this handler. An * optimization FIXME, I guess. */ movi a3, PS_INTLEVEL_MASK and a3, a0, a3 bnez a3, _not_l1 /* interrupt masking is zero, so no need to zero it before OR-ing. */ movi a3, PS_INTLEVEL(1) or a0, a0, a3 _not_l1: wsr.ZSR_EPS a0 #endif /* CONFIG_XTENSA_INTERRUPT_NONPREEMPTABLE */ movi a3, PS_INTLEVEL(0xf) or a0, a0, a3 wsr.ps a0 rsync /* A1 already contains our saved stack, and A2 our handler. * So all that's needed for CROSS_STACK_CALL is to put the * "new" stack into A3. This can be either a copy of A1 or an * entirely new area depending on whether we find a 1 in our * SR[off] macro argument. */ rsr.ZSR_CPU a3 l32i a0, a3, \NEST_OFF beqz a0, _switch_stacks_\@ /* Use the same stack, just copy A1 to A3 after incrementing NEST */ addi a0, a0, 1 s32i a0, a3, \NEST_OFF mov a3, a1 j _do_call_\@ _switch_stacks_\@: addi a0, a0, 1 s32i a0, a3, \NEST_OFF l32i a3, a3, \INTSTACK_OFF _do_call_\@: CROSS_STACK_CALL /* Mask interrupts (which have been unmasked during the handler * execution) while we muck with the windows and decrement the nested * count. The restore will unmask them correctly. */ rsil a0, XCHAL_NUM_INTLEVELS /* Decrement nest count */ rsr.ZSR_CPU a3 l32i a0, a3, \NEST_OFF addi a0, a0, -1 s32i a0, a3, \NEST_OFF /* Last trick: the called function returned the "next" handle * to restore to in A6 (the call4'd function's A2). If this * is not the same handle as we started with, we need to do a * register spill before restoring, for obvious reasons. * Remember to restore the A1 stack pointer as it existed at * interrupt time so the caller of the interrupted function * spills to the right place. */ beq a6, a1, _restore_\@ #if !defined(CONFIG_KERNEL_COHERENCE) || \ (defined(CONFIG_KERNEL_COHERENCE) && defined(CONFIG_SCHED_CPU_MASK_PIN_ONLY)) l32i a1, a1, 0 l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET addi a1, a1, ___xtensa_irq_bsa_t_SIZEOF /* When using coherence, the registers of the interrupted * context got spilled upstream in arch_cohere_stacks() */ SPILL_ALL_WINDOWS #endif #if defined(CONFIG_KERNEL_COHERENCE) && \ defined(CONFIG_USERSPACE) && \ !defined(CONFIG_SCHED_CPU_MASK_PIN_ONLY) /* With userspace enabled, we need to swap page table via function calls * above after returning from syscall handler above in CROSS_STACK_CALL. * This means that the stack is being actively used, and so we need to * flush the cached data in stack. */ movi a2, 0 xsr.ZSR_FLUSH a2 beqz a2, _excint_noflush_\@ rsr.ZSR_CPU a3 l32i a3, a3, \NEST_OFF bnez a3, _excint_noflush_\@ mov a3, a1 _excint_flushloop_\@: dhwb a3, 0 addi a3, a3, XCHAL_DCACHE_LINESIZE blt a3, a2, _excint_flushloop_\@ _excint_noflush_\@: #endif /* CONFIG_KERNEL_COHERENCE && CONFIG_USERSPACE && !CONFIG_SCHED_CPU_MASK_PIN_ONLY */ /* Restore A1 stack pointer from "next" handle. */ mov a1, a6 _restore_\@: j _restore_context .endm /* Defines an exception/interrupt vector for a specified level. Saves * off the interrupted A0-A3 registers and the per-level PS/PC * registers to the stack before jumping to a handler (defined with * EXCINT_HANDLER) to do the rest of the work. * * Arguments are a numeric interrupt level and symbol names for the * entry code (defined via EXCINT_HANDLER) and a C handler for this * particular level. * * Note that the linker sections for some levels get special names for * no particularly good reason. Only level 1 has any code generation * difference, because it is the legacy exception level that predates * the EPS/EPC registers. It also lives in the "iram0.text" segment * (which is linked immediately after the vectors) so that an assembly * stub can be loaded into the vector area instead and reach this code * with a simple jump instruction. */ .macro DEF_EXCINT LVL, ENTRY_SYM, C_HANDLER_SYM #if defined(CONFIG_XTENSA_SMALL_VECTOR_TABLE_ENTRY) .pushsection .iram.text, "ax" .global _Level\LVL\()VectorHelper _Level\LVL\()VectorHelper : #else .if \LVL == 1 .pushsection .iram0.text, "ax" .elseif \LVL == XCHAL_DEBUGLEVEL .pushsection .DebugExceptionVector.text, "ax" .elseif \LVL == XCHAL_NMILEVEL .pushsection .NMIExceptionVector.text, "ax" .else .pushsection .Level\LVL\()InterruptVector.text, "ax" .endif .global _Level\LVL\()Vector _Level\LVL\()Vector: #endif #ifdef CONFIG_XTENSA_MMU .if \LVL == 1 /* If there are any TLB misses during interrupt handling, * the user/kernel/double exception vector will be triggered * to handle these misses. This results in DEPC and EXCCAUSE * being overwritten, and then execution returned back to * this site of TLB misses. When it gets to the C handler, * it will not see the original cause. So stash * the EXCCAUSE here so C handler can see the original cause. * * For double exception, DEPC in saved in earlier vector * code. */ wsr a0, ZSR_EXCCAUSE_SAVE esync rsr a0, ZSR_DEPC_SAVE beqz a0, _not_triple_fault /* If stashed DEPC is not zero, we have started servicing * a double exception and yet we are here because there is * another exception (through user/kernel if PS.EXCM is * cleared, or through double if PS.EXCM is set). This can * be considered triple fault. Although there is no triple * faults on Xtensa. Once PS.EXCM is set, it keeps going * through double exception vector for any new exceptions. * However, our exception code needs to unmask PS.EXCM to * enable register window operations. So after that, any * new exceptions will go through the kernel or user vectors * depending on PS.UM. If there is continuous faults, it may * keep ping-ponging between double and kernel/user exception * vectors that may never get resolved. Since we stash DEPC * during double exception, and the stashed one is only cleared * once the double exception has been processed, we can use * the stashed DEPC value to detect if the next exception could * be considered a triple fault. If such a case exists, simply * jump to an infinite loop, or quit the simulator, or invoke * debugger. */ rsr a0, ZSR_EXCCAUSE_SAVE j _TripleFault _not_triple_fault: rsr.exccause a0 xsr a0, ZSR_EXCCAUSE_SAVE esync .endif #endif addi a1, a1, -___xtensa_irq_bsa_t_SIZEOF s32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET s32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET s32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET /* Level "1" is the exception handler, which uses a different * calling convention. No special register holds the * interrupted PS, instead we just assume that the CPU has * turned on the EXCM bit and set INTLEVEL. */ .if \LVL == 1 rsr.ps a0 #ifdef CONFIG_XTENSA_MMU /* TLB misses also come through level 1 interrupts. * We do not want to unconditionally unmask interrupts. * Execution continues after a TLB miss is handled, * and we need to preserve the interrupt mask. * The interrupt mask will be cleared for non-TLB-misses * level 1 interrupt later in the handler code. */ movi a2, ~PS_EXCM_MASK #else movi a2, ~(PS_EXCM_MASK | PS_INTLEVEL_MASK) #endif and a0, a0, a2 s32i a0, a1, ___xtensa_irq_bsa_t_ps_OFFSET .else rsr.eps\LVL a0 s32i a0, a1, ___xtensa_irq_bsa_t_ps_OFFSET .endif #ifdef CONFIG_USERSPACE /* When restoring context via xtensa_switch and * returning from non-nested interrupts, we will be * using the stashed PS value in the thread struct * instead of the one in the thread stack. Both of * these scenarios will have nested value of 0. * So when nested value is zero, we store the PS * value into thread struct. */ rsr.ZSR_CPU a3 l32i a2, a3, ___cpu_t_nested_OFFSET bnez a2, _excint_skip_ps_save_to_thread_\LVL l32i a2, a3, ___cpu_t_current_OFFSET s32i a0, a2, _thread_offset_to_return_ps _excint_skip_ps_save_to_thread_\LVL: #endif rsr.epc\LVL a0 s32i a0, a1, ___xtensa_irq_bsa_t_pc_OFFSET /* What's happening with this jump is that the L32R * instruction to load a full 32 bit immediate must use an * offset that is negative from PC. Normally the assembler * fixes this up for you by putting the "literal pool" * somewhere at the start of the section. But vectors start * at a fixed address in their own section, and don't (in our * current linker setup) have anywhere "definitely before * vectors" to place immediates. Some platforms and apps will * link by dumb luck, others won't. We add an extra jump just * to clear space we know to be legal. * * The right way to fix this would be to use a "literal_prefix" * to put the literals into a per-vector section, then link * that section into the PREVIOUS vector's area right after * the vector code. Requires touching a lot of linker scripts * though. */ j _after_imms\LVL\() .align 4 _handle_excint_imm\LVL: .word \ENTRY_SYM _c_handler_imm\LVL: .word \C_HANDLER_SYM _after_imms\LVL: l32r a2, _c_handler_imm\LVL l32r a0, _handle_excint_imm\LVL jx a0 .popsection #if defined(CONFIG_XTENSA_SMALL_VECTOR_TABLE_ENTRY) .if \LVL == 1 .pushsection .iram0.text, "ax" .elseif \LVL == XCHAL_DEBUGLEVEL .pushsection .DebugExceptionVector.text, "ax" .elseif \LVL == XCHAL_NMILEVEL .pushsection .NMIExceptionVector.text, "ax" .else .pushsection .Level\LVL\()InterruptVector.text, "ax" .endif .global _Level\LVL\()Vector _Level\LVL\()Vector : j _Level\LVL\()VectorHelper .popsection #endif .endm #endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_XTENSA_ASM2_S_H */