arch/xtensa: soc/xtensa/intel_adsp: Enable KERNEL_COHERENCE

Implement the kernel "coherence" API on top of the linker
cached/uncached mapping work.

Add Xtensa handling for the stack coherence API.

Signed-off-by: Andy Ross <andrew.j.ross@intel.com>

arch/xtensa/core/xtensa-asm2-util.S

@@ -210,6 +210,18 @@ xtensa_switch:
 	/* Now the high registers */
 	call0 xtensa_save_high_regs
 
+#ifdef CONFIG_KERNEL_COHERENCE
+	/* Flush the stack.  The top of stack was stored for us in
+	 * EXCSAVE3 (FIXME: shouldn't be hardcoded!) by arch_cohere_stacks().
+	 */
+	rsr.EXCSAVE3 a0
+	mov a3, a1
+flushloop:
+	dhwb a3, 0
+	addi a3, a3, XCHAL_DCACHE_LINESIZE
+	blt a3, a0, flushloop
+#endif
+
 	/* Restore the A3 argument we spilled earlier (via the base
 	 * save pointer pushed at the bottom of the stack) and set the
 	 * stack to the "new" context out of the A2 spill slot.

arch/xtensa/core/xtensa-asm2.c

@@ -27,7 +27,7 @@ void *xtensa_init_stack(int *stack_top,
 	 * start will decrement the stack pointer by 16.
 	 */
 	const int bsasz = BASE_SAVE_AREA_SIZE - 16;
-	void **bsa = (void **) (((char *) stack_top) - bsasz);
+	void *ret, **bsa = (void **) (((char *) stack_top) - bsasz);
 
 	(void)memset(bsa, 0, bsasz);
 
@@ -53,7 +53,12 @@ void *xtensa_init_stack(int *stack_top,
 	 * as the handle
 	 */
 	bsa[-9] = bsa;
-	return &bsa[-9];
+	ret = &bsa[-9];
+
+#ifdef CONFIG_KERNEL_COHERENCE
+	xthal_dcache_region_writeback(ret, (char *)stack_top - (char *)ret);
+#endif
+	return ret;
 }
 
 void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,

arch/xtensa/include/kernel_arch_func.h

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2016 Wind River Systems, Inc.
  * Copyright (c) 2016 Cadence Design Systems, Inc.
+ * Copyright (c) 2020 Intel Corporation
  * SPDX-License-Identifier: Apache-2.0
  */
 
@@ -32,6 +33,13 @@ static ALWAYS_INLINE void arch_kernel_init(void)
 {
 	_cpu_t *cpu0 = &_kernel.cpus[0];
 
+#ifdef CONFIG_KERNEL_COHERENCE
+	/* Make sure we don't have live data for unexpected cached
+	 * regions due to boot firmware
+	 */
+	xthal_dcache_all_writeback_inv();
+#endif
+
 	cpu0->nested = 0;
 
 	/* The asm2 scheme keeps the kernel pointer in MISC0 for easy
@@ -55,6 +63,60 @@ static inline void arch_switch(void *switch_to, void **switched_from)
 	return xtensa_switch(switch_to, switched_from);
 }
 
+/* FIXME: we don't have a framework for including this from the SoC
+ * layer, so we define it in the arch code here.
+ */
+#ifdef CONFIG_SOC_FAMILY_INTEL_ADSP
+static inline bool arch_mem_coherent(void *ptr)
+{
+    size_t addr = (size_t) ptr;
+
+    return addr >= 0x80000000 && addr < 0xa0000000;
+}
+#endif
+
+#ifdef CONFIG_KERNEL_COHERENCE
+static inline void arch_cohere_stacks(struct k_thread *old_thread,
+				      void *old_switch_handle,
+				      struct k_thread *new_thread)
+{
+	size_t ostack = old_thread->stack_info.start;
+	size_t osz    = old_thread->stack_info.size;
+	size_t osp    = (size_t) old_switch_handle;
+
+	size_t nstack = new_thread->stack_info.start;
+	size_t nsz    = new_thread->stack_info.size;
+	size_t nsp    = (size_t) new_thread->switch_handle;
+
+	xthal_dcache_region_invalidate((void *)nsp, (nstack + nsz) - nsp);
+
+	/* FIXME: dummy initializion threads don't have stack info set
+	 * up and explode the logic above.  Find a way to get this
+	 * test out of the hot paths!
+	 */
+	if (old_thread->base.thread_state & _THREAD_DUMMY) {
+		return;
+	}
+
+	/* In interrupt context, we have a valid frame already from
+	 * the interrupt entry code, but for arch_switch() that hasn't
+	 * happened yet.  It will do the flush itself, we just have to
+	 * calculate the boundary for it.
+	 */
+	if (old_switch_handle != NULL) {
+		xthal_dcache_region_writeback((void *)osp,
+					      (ostack + osz) - osp);
+	} else {
+		/* FIXME: hardcoding EXCSAVE3 is bad, should be
+		 * configurable a-la XTENSA_KERNEL_CPU_PTR_SR.
+		 */
+		uint32_t end = ostack + osz;
+
+		__asm__ volatile("wsr.EXCSAVE3 %0" :: "r"(end));
+	}
+}
+#endif
+
 #ifdef __cplusplus
 }
 #endif
@@ -63,6 +125,7 @@ static inline bool arch_is_in_isr(void)
 {
 	return arch_curr_cpu()->nested != 0U;
 }
+
 #endif /* _ASMLANGUAGE */
 
 #endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_KERNEL_ARCH_FUNC_H_ */

include/arch/xtensa/arch.h

@@ -28,7 +28,11 @@
 #include <xtensa/config/core.h>
 #include <arch/common/addr_types.h>
 
+#ifdef CONFIG_KERNEL_COHERENCE
+#define ARCH_STACK_PTR_ALIGN XCHAL_DCACHE_LINESIZE
+#else
 #define ARCH_STACK_PTR_ALIGN 16
+#endif
 
 /* Xtensa GPRs are often designated by two different names */
 #define sys_define_gpr_with_alias(name1, name2) union { uint32_t name1, name2; }

soc/xtensa/intel_adsp/Kconfig

@@ -5,6 +5,7 @@
 
 config SOC_FAMILY_INTEL_ADSP
 	bool
+	select ARCH_HAS_COHERENCE
 
 if SOC_FAMILY_INTEL_ADSP
 

soc/xtensa/intel_adsp/cavs_v15/linker.ld

@@ -202,16 +202,7 @@ _memmap_cacheattr_bp_allvalid = 0x22222222;
  * as cacheattr_set macro sets them both to the same set of
  * attributes.
  */
-#ifndef CONFIG_SMP
 _memmap_cacheattr_intel_cavs15_adsp = 0xFF42FFF2;
-#else
-/*
- * FIXME: Make 0xA0000000 - 0xBFFFFFFF to bypass cache under SMP
- * since there is no data cache manipulation for spinlock, kernel
- * object, scheduler, etc...
- */
-_memmap_cacheattr_intel_cavs15_adsp = 0xFF22FFF2;
-#endif
 
 PROVIDE(_memmap_cacheattr_reset = _memmap_cacheattr_intel_cavs15_adsp);
 SECTIONS
@@ -426,7 +417,7 @@ SECTIONS
     *(.noinit.*)
   } >ram :ram_phdr
 
-  .lit4 SEGSTART_CACHED : ALIGN(4)
+  .lit4 : ALIGN(4)
   {
     _lit4_start = ABSOLUTE(.);
     *(*.lit4)
@@ -466,11 +457,13 @@ SECTIONS
 #define ROMABLE_REGION ucram :ucram_phdr
 #include <linker/common-ram.ld>
 
-  /* ANDY TEST */
+  /* This section is cached.  By default it contains only declared
-  .ucram SEGSTART_UNCACHED :
+   * thread stacks, but applications can put symbols here too.
+   */
+  .cached SEGSTART_CACHED :
   {
-    *(.ucram)
+    *(.cached .cached.*)
-  } >ucram :ucram_phdr
+  } >ram :ram_phdr
 
   .bss SEGSTART_UNCACHED (NOLOAD) : ALIGN(4096)
   {

soc/xtensa/intel_adsp/common/soc_mp.c

@@ -79,6 +79,11 @@ static void mp_entry2(void)
 	volatile int ie;
 	uint32_t idc_reg;
 
+	/* We don't know what the boot ROM might have touched and we
+	 * don't care.  Make sure it's not in our local cache.
+	 */
+	xthal_dcache_all_writeback_inv();
+
 	/* Copy over VECBASE from the main CPU for an initial value
 	 * (will need to revisit this if we ever allow a user API to
 	 * change interrupt vectors at runtime).