arch/xtensa: Remember to spill windows in arch_cohere_stacks()

When we reach this code in interrupt context, our upper GPRs contain a
cross-stack call that may still include some registers from the
interrupted thread.  Those need to go out to memory before we can do
our cache coherence dance here.

Signed-off-by: Andy Ross <andrew.j.ross@intel.com>

arch/xtensa/core/xtensa-asm2-util.S

@@ -10,12 +10,11 @@
 /*
  * xtensa_spill_reg_windows
  *
- * Globally visible symbol to do register spills.  Useful for unit
+ * Spill all register windows.  Not a C function, enter this via CALL0
- * testing, or maybe as part of a debug/watchdog/error handler.  Not a
+ * (so you have to save off A0, but no other registers need to be
- * C function, call this via CALL0 (so you probably have to save off
+ * spilled).  On return, all registers not part of the current
- * A0, but no other registers need to be spilled).  On return, all
+ * function will be spilled to memory.  The WINDOWSTART SR will have a
- * registers not part of the current function will be spilled to
+ * single 1 bit corresponding to the current frame at WINDOWBASE.
- * memory.
  */
 .global xtensa_spill_reg_windows
 .align 4

arch/xtensa/include/kernel_arch_func.h

@@ -83,6 +83,15 @@ static ALWAYS_INLINE void arch_cohere_stacks(struct k_thread *old_thread,
 	size_t nsz    = new_thread->stack_info.size;
 	size_t nsp    = (size_t) new_thread->switch_handle;
 
+	if (old_switch_handle != NULL) {
+		int32_t a0save;
+
+		__asm__ volatile("mov %0, a0;"
+				 "call0 xtensa_spill_reg_windows;"
+				 "mov a0, %0"
+				 : "=r"(a0save));
+	}
+
 	/* The "live" area (the region between the switch handle,
 	 * which is the stack pointer, and the top of the stack
 	 * memory) of the inbound stack needs to be invalidated: it

arch/xtensa/include/xtensa-asm2-s.h

@@ -326,7 +326,12 @@ _do_call_\@:
 	l32i a1, a1, 0
 	l32i a0, a1, BSA_A0_OFF
 	addi a1, a1, BASE_SAVE_AREA_SIZE
+#ifndef CONFIG_KERNEL_COHERENCE
+	/* When using coherence, the registers of the interrupted
+	 * context got spilled upstream in arch_cohere_stacks()
+	 */
 	SPILL_ALL_WINDOWS
+#endif
 	mov a1, a6
 
 _restore_\@: