From 0e83961b21d5740c3865eaa615ca3c274d9b8a3f Mon Sep 17 00:00:00 2001 From: Andy Ross Date: Tue, 12 May 2020 14:27:18 -0700 Subject: [PATCH] arch/xtensa: soc/xtensa/intel_adsp: Enable KERNEL_COHERENCE Implement the kernel "coherence" API on top of the linker cached/uncached mapping work. Add Xtensa handling for the stack coherence API. Signed-off-by: Andy Ross --- arch/xtensa/core/xtensa-asm2-util.S | 12 +++++ arch/xtensa/core/xtensa-asm2.c | 9 +++- arch/xtensa/include/kernel_arch_func.h | 63 ++++++++++++++++++++++++ include/arch/xtensa/arch.h | 4 ++ soc/xtensa/intel_adsp/Kconfig | 1 + soc/xtensa/intel_adsp/cavs_v15/linker.ld | 21 +++----- soc/xtensa/intel_adsp/common/soc_mp.c | 5 ++ 7 files changed, 99 insertions(+), 16 deletions(-) diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index 5e1b7672ff6..995861e2fa8 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -210,6 +210,18 @@ xtensa_switch: /* Now the high registers */ call0 xtensa_save_high_regs +#ifdef CONFIG_KERNEL_COHERENCE + /* Flush the stack. The top of stack was stored for us in + * EXCSAVE3 (FIXME: shouldn't be hardcoded!) by arch_cohere_stacks(). + */ + rsr.EXCSAVE3 a0 + mov a3, a1 +flushloop: + dhwb a3, 0 + addi a3, a3, XCHAL_DCACHE_LINESIZE + blt a3, a0, flushloop +#endif + /* Restore the A3 argument we spilled earlier (via the base * save pointer pushed at the bottom of the stack) and set the * stack to the "new" context out of the A2 spill slot. diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index 64f1b2ca5a8..05b6b6c30e1 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -27,7 +27,7 @@ void *xtensa_init_stack(int *stack_top, * start will decrement the stack pointer by 16. */ const int bsasz = BASE_SAVE_AREA_SIZE - 16; - void **bsa = (void **) (((char *) stack_top) - bsasz); + void *ret, **bsa = (void **) (((char *) stack_top) - bsasz); (void)memset(bsa, 0, bsasz); @@ -53,7 +53,12 @@ void *xtensa_init_stack(int *stack_top, * as the handle */ bsa[-9] = bsa; - return &bsa[-9]; + ret = &bsa[-9]; + +#ifdef CONFIG_KERNEL_COHERENCE + xthal_dcache_region_writeback(ret, (char *)stack_top - (char *)ret); +#endif + return ret; } void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, diff --git a/arch/xtensa/include/kernel_arch_func.h b/arch/xtensa/include/kernel_arch_func.h index a7b0d68f6bb..5521416956b 100644 --- a/arch/xtensa/include/kernel_arch_func.h +++ b/arch/xtensa/include/kernel_arch_func.h @@ -1,6 +1,7 @@ /* * Copyright (c) 2016 Wind River Systems, Inc. * Copyright (c) 2016 Cadence Design Systems, Inc. + * Copyright (c) 2020 Intel Corporation * SPDX-License-Identifier: Apache-2.0 */ @@ -32,6 +33,13 @@ static ALWAYS_INLINE void arch_kernel_init(void) { _cpu_t *cpu0 = &_kernel.cpus[0]; +#ifdef CONFIG_KERNEL_COHERENCE + /* Make sure we don't have live data for unexpected cached + * regions due to boot firmware + */ + xthal_dcache_all_writeback_inv(); +#endif + cpu0->nested = 0; /* The asm2 scheme keeps the kernel pointer in MISC0 for easy @@ -55,6 +63,60 @@ static inline void arch_switch(void *switch_to, void **switched_from) return xtensa_switch(switch_to, switched_from); } +/* FIXME: we don't have a framework for including this from the SoC + * layer, so we define it in the arch code here. + */ +#ifdef CONFIG_SOC_FAMILY_INTEL_ADSP +static inline bool arch_mem_coherent(void *ptr) +{ + size_t addr = (size_t) ptr; + + return addr >= 0x80000000 && addr < 0xa0000000; +} +#endif + +#ifdef CONFIG_KERNEL_COHERENCE +static inline void arch_cohere_stacks(struct k_thread *old_thread, + void *old_switch_handle, + struct k_thread *new_thread) +{ + size_t ostack = old_thread->stack_info.start; + size_t osz = old_thread->stack_info.size; + size_t osp = (size_t) old_switch_handle; + + size_t nstack = new_thread->stack_info.start; + size_t nsz = new_thread->stack_info.size; + size_t nsp = (size_t) new_thread->switch_handle; + + xthal_dcache_region_invalidate((void *)nsp, (nstack + nsz) - nsp); + + /* FIXME: dummy initializion threads don't have stack info set + * up and explode the logic above. Find a way to get this + * test out of the hot paths! + */ + if (old_thread->base.thread_state & _THREAD_DUMMY) { + return; + } + + /* In interrupt context, we have a valid frame already from + * the interrupt entry code, but for arch_switch() that hasn't + * happened yet. It will do the flush itself, we just have to + * calculate the boundary for it. + */ + if (old_switch_handle != NULL) { + xthal_dcache_region_writeback((void *)osp, + (ostack + osz) - osp); + } else { + /* FIXME: hardcoding EXCSAVE3 is bad, should be + * configurable a-la XTENSA_KERNEL_CPU_PTR_SR. + */ + uint32_t end = ostack + osz; + + __asm__ volatile("wsr.EXCSAVE3 %0" :: "r"(end)); + } +} +#endif + #ifdef __cplusplus } #endif @@ -63,6 +125,7 @@ static inline bool arch_is_in_isr(void) { return arch_curr_cpu()->nested != 0U; } + #endif /* _ASMLANGUAGE */ #endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_KERNEL_ARCH_FUNC_H_ */ diff --git a/include/arch/xtensa/arch.h b/include/arch/xtensa/arch.h index 553c8abe05d..b0a01945fd8 100644 --- a/include/arch/xtensa/arch.h +++ b/include/arch/xtensa/arch.h @@ -28,7 +28,11 @@ #include #include +#ifdef CONFIG_KERNEL_COHERENCE +#define ARCH_STACK_PTR_ALIGN XCHAL_DCACHE_LINESIZE +#else #define ARCH_STACK_PTR_ALIGN 16 +#endif /* Xtensa GPRs are often designated by two different names */ #define sys_define_gpr_with_alias(name1, name2) union { uint32_t name1, name2; } diff --git a/soc/xtensa/intel_adsp/Kconfig b/soc/xtensa/intel_adsp/Kconfig index 65d29b896b2..5a7234e956c 100644 --- a/soc/xtensa/intel_adsp/Kconfig +++ b/soc/xtensa/intel_adsp/Kconfig @@ -5,6 +5,7 @@ config SOC_FAMILY_INTEL_ADSP bool + select ARCH_HAS_COHERENCE if SOC_FAMILY_INTEL_ADSP diff --git a/soc/xtensa/intel_adsp/cavs_v15/linker.ld b/soc/xtensa/intel_adsp/cavs_v15/linker.ld index 7d7ddda5195..9feb781db31 100644 --- a/soc/xtensa/intel_adsp/cavs_v15/linker.ld +++ b/soc/xtensa/intel_adsp/cavs_v15/linker.ld @@ -202,16 +202,7 @@ _memmap_cacheattr_bp_allvalid = 0x22222222; * as cacheattr_set macro sets them both to the same set of * attributes. */ -#ifndef CONFIG_SMP _memmap_cacheattr_intel_cavs15_adsp = 0xFF42FFF2; -#else -/* - * FIXME: Make 0xA0000000 - 0xBFFFFFFF to bypass cache under SMP - * since there is no data cache manipulation for spinlock, kernel - * object, scheduler, etc... - */ -_memmap_cacheattr_intel_cavs15_adsp = 0xFF22FFF2; -#endif PROVIDE(_memmap_cacheattr_reset = _memmap_cacheattr_intel_cavs15_adsp); SECTIONS @@ -426,7 +417,7 @@ SECTIONS *(.noinit.*) } >ram :ram_phdr - .lit4 SEGSTART_CACHED : ALIGN(4) + .lit4 : ALIGN(4) { _lit4_start = ABSOLUTE(.); *(*.lit4) @@ -466,11 +457,13 @@ SECTIONS #define ROMABLE_REGION ucram :ucram_phdr #include - /* ANDY TEST */ - .ucram SEGSTART_UNCACHED : + /* This section is cached. By default it contains only declared + * thread stacks, but applications can put symbols here too. + */ + .cached SEGSTART_CACHED : { - *(.ucram) - } >ucram :ucram_phdr + *(.cached .cached.*) + } >ram :ram_phdr .bss SEGSTART_UNCACHED (NOLOAD) : ALIGN(4096) { diff --git a/soc/xtensa/intel_adsp/common/soc_mp.c b/soc/xtensa/intel_adsp/common/soc_mp.c index 6b05de4c9ff..ee6792fdda0 100644 --- a/soc/xtensa/intel_adsp/common/soc_mp.c +++ b/soc/xtensa/intel_adsp/common/soc_mp.c @@ -79,6 +79,11 @@ static void mp_entry2(void) volatile int ie; uint32_t idc_reg; + /* We don't know what the boot ROM might have touched and we + * don't care. Make sure it's not in our local cache. + */ + xthal_dcache_all_writeback_inv(); + /* Copy over VECBASE from the main CPU for an initial value * (will need to revisit this if we ever allow a user API to * change interrupt vectors at runtime).