arch/x86_64: Add a per-CPU SMP bringup API

The previous scheme where the xuk layer would call out to the to
"fetch" the stack for a SMP CPU at startup was sorta weird, and an
impedance mismatch with Zephyr which has a "start this CPU" call
instead.  It also got broken when x86_64 started launching CPUs
(correctly) on their interrupt stacks instead of a temporary area;
they weren't ready yet when xuk initialization was happening and the
system would deadlock waiting for code that can't run yet to provide a
stack.

Note that this preserves the somewhat quirky behavior that Zephyr's
CPU numbering is just the order in which the SMP CPUs emerge from
initialization and not a hardware ID.

Signed-off-by: Andy Ross <andrew.j.ross@intel.com>

arch/x86_64/core/demo-kernel.c

@@ -46,11 +46,6 @@ void test_timers(void)
 	printf("tsc %d apic %d\n", tsc1 - tsc0, apic0 - apic1);
 }
 
-unsigned int _init_cpu_stack(int cpu)
-{
-	return (long)alloc_page(0) + 4096;
-}
-
 void handler_timer(void *arg, int err)
 {
 	printf("Timer expired on CPU%d\n", (int)(long)xuk_get_f_ptr());
@@ -159,6 +154,7 @@ void _cpu_start(int cpu)
 	test_timers();
 
 	if (cpu == 0) {
+		xuk_start_cpu(1, (long)alloc_page(0) + 4096);
 		xuk_set_isr(0x1f3, 0, (void *)handler_f3, (void *)0x12345678);
 	}
 

arch/x86_64/core/x86_64.c

@@ -72,15 +72,15 @@ void *_isr_exit_restore_stack(void *interrupted)
 struct {
 	void (*fn)(int, void*);
 	void *arg;
-	unsigned int esp;
 } cpu_init[CONFIG_MP_NUM_CPUS];
 
 /* Called from Zephyr initialization */
 void z_arch_start_cpu(int cpu_num, k_thread_stack_t *stack, int sz,
 		     void (*fn)(int, void *), void *arg)
 {
+	xuk_start_cpu(cpu_num, (int)(sz + (char *)stack));
+
 	cpu_init[cpu_num].arg = arg;
-	cpu_init[cpu_num].esp = (int)(long)(sz + (char *)stack);
 
 	/* This is our flag to the spinning CPU.  Do this last */
 	cpu_init[cpu_num].fn = fn;
@@ -155,18 +155,9 @@ void _cpu_start(int cpu)
 	}
 }
 
-/* Returns the initial stack to use for CPU startup on auxiliary (not
- * cpu 0) processors to the xuk layer, which gets selected by the
- * non-arch Zephyr kernel and stashed by z_arch_start_cpu()
- */
-unsigned int _init_cpu_stack(int cpu)
-{
-	return cpu_init[cpu].esp;
-}
-
 int z_arch_irq_connect_dynamic(unsigned int irq, unsigned int priority,
-			      void (*routine)(void *parameter), void *parameter,
+			       void (*routine)(void *parameter), void *parameter,
-			      u32_t flags)
+			       u32_t flags)
 {
 	ARG_UNUSED(flags);
 	__ASSERT(priority >= 2 && priority <= 15,

arch/x86_64/core/xuk.c

@@ -492,15 +492,17 @@ static void smp_init(void)
 	};
 	while (_apic.ICR_LO.send_pending) {
 	}
+}
 
-	for (int i = 1; i < CONFIG_MP_NUM_CPUS; i++) {
+void xuk_start_cpu(int cpu, unsigned int stack)
-		_shared.smpinit_stack = _init_cpu_stack(i);
+{
-		printf("Granting stack @ %xh to CPU %d\n",
+	int act = _shared.num_active_cpus;
-		       _shared.smpinit_stack, i);
 
-		while (_shared.num_active_cpus <= i) {
+	printf("Granting stack @ %xh to CPU %d\n", stack, cpu);
-			__asm__("pause");
+	_shared.smpinit_stack = stack;
-		}
+
+	while (_shared.num_active_cpus == act) {
+		__asm__ volatile("pause");
 	}
 }
 

arch/x86_64/core/xuk.h

@@ -138,15 +138,16 @@ struct xuk_stack_frame {
 long xuk_setup_stack(long sp, void *fn, unsigned int eflags,
 		     long *args, int nargs);
 
+
+/* Starts the numbered CPU running with the specified initial stack
+ * pointer
+ */
+
+void xuk_start_cpu(int cpu, unsigned int stack);
 /*
  * OS-defined utilities required by the xuk layer:
  */
 
-/* Returns the address of a stack pointer in 32 bit memory to be used
- * by AP processor bootstraping and startup.
- */
-unsigned int _init_cpu_stack(int cpu);
-
 /* OS CPU startup entry point, running on the stack returned by
  * init_cpu_stack()
  */