kernel: SMP timer integration

In SMP, the system timer is used for timeslicing on auxiliary CPUs,
but the base system timekeeping via _nano_sys_clock_tick_announce() is
still done on CPU0 only (because the framework isn't prepared for
asynchronous notification yet).  Skip processing on CPU1+.

Also, due to a hardware interaction* that is difficult to work around,
timer initialization on the auxiliary CPUs is done at the very end of
the CPU bringup, just before the swap into the scheduler.  A
smp_timer_init() API has been added for this purpose.

* On ESP-32, enabling the timer seems to result in a near-synchronous
  interrupt being delivered despite my best attempts to keep it
  masked, then blowing things up because the CPU record isn't set up
  to handle it yet.

Signed-off-by: Andy Ross <andrew.j.ross@intel.com>

drivers/timer/xtensa_sys_timer.c

@@ -11,6 +11,7 @@
 #include <xtensa/tie/xt_timer.h>
 
 #include <xtensa_timer.h>
+#include <kernel_structs.h>
 #include "irq.h"
 
 #ifdef XT_BOARD
@@ -444,6 +445,20 @@ void _zxt_tick_timer_init(void)
 }
 #endif
 
+#ifdef CONFIG_SMP
+/**
+ * @brief Timer initialization for SMP auxiliary CPUs
+ *
+ * Called on MP CPUs other than zero.  Some architectures appear to
+ * generate spurious timer interrupts during initialization, so this
+ * function must be called late in the SMP initialization sequence.
+ */
+void smp_timer_init(void)
+{
+	_zxt_tick_timer_init();
+}
+#endif
+
 /*
  * Compute and initialize at run-time the tick divisor (the number of
  * processor clock cycles in an RTOS tick, used to set the tick timer).
@@ -496,6 +511,17 @@ void _timer_int_handler(void *params)
 	_sys_k_event_logger_interrupt();
 #endif
 
+#ifdef CONFIG_SMP
+	/* The timer infractructure isn't prepared to handle
+	 * asynchronous timeouts on multiple CPUs.  In SMP we use the
+	 * timer interrupt on auxiliary CPUs only for scheduling.
+	 * Don't muck up the timeout math.
+	 */
+	if (_arch_curr_cpu()->id) {
+		return;
+	}
+#endif
+
 #ifdef CONFIG_TICKLESS_KERNEL
 	if (!idle_original_ticks) {
 		_set_max_clock_time();

kernel/include/kernel_internal.h

@@ -176,6 +176,8 @@ extern void _thread_monitor_exit(struct k_thread *thread);
 
 extern void smp_init(void);
 
+extern void smp_timer_init(void);
+
 #ifdef __cplusplus
 }
 #endif

kernel/smp.c

@@ -73,8 +73,10 @@ static void _smp_init_top(int key, void *arg)
 		.base.thread_state = _THREAD_DUMMY,
 	};
 
+	int k = irq_lock();
 	_arch_curr_cpu()->current = &dummy_thread;
-	_Swap(irq_lock());
+	smp_timer_init();
+	_Swap(k);
 
 	CODE_UNREACHABLE;
 }

kernel/sys_clock.c

@@ -317,6 +317,13 @@ static void handle_time_slicing(s32_t ticks)
  */
 void _nano_sys_clock_tick_announce(s32_t ticks)
 {
+#ifdef CONFIG_SMP
+	/* sys_clock timekeeping happens only on the main CPU */
+	if (_arch_curr_cpu()->id) {
+		return;
+	}
+#endif
+
 #ifndef CONFIG_TICKLESS_KERNEL
 	unsigned int  key;