x86: implement direct interrupts

Change-Id: Icac461a361dde969f023e7aa11f0605a97a3c009
Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>

arch/x86/core/intstub.S

@@ -257,7 +257,7 @@ alreadyOnIntStack:
 #endif
 
 	/* irq_controller.h interface */
-	_irq_controller_eoi
+	_irq_controller_eoi_macro
 
 #ifdef CONFIG_INT_LATENCY_BENCHMARK
 	call	_int_latency_start

arch/x86/core/irq_manage.c

@@ -20,6 +20,7 @@
 #include <misc/__assert.h>
 #include <misc/printk.h>
 #include <irq.h>
+#include <logging/kernel_event_logger.h>
 
 extern void _SpuriousIntHandler(void *);
 extern void _SpuriousIntNoErrCodeHandler(void *);
@@ -45,3 +46,68 @@ void *__attribute__((section(".spurNoErrIsr")))
 	MK_ISR_NAME(_SpuriousIntNoErrCodeHandler) =
 		&_SpuriousIntNoErrCodeHandler;
 
+
+/* FIXME: IRQ direct inline functions have to be placed here and not in
+ * arch/cpu.h as inline functions due to nasty circular dependency between
+ * arch/cpu.h and kernel_structs.h; the inline functions typically need to
+ * perform operations on _kernel.  For now, leave as regular functions, a
+ * future iteration will resolve this.
+ * We have a similar issue with the k_event_logger functions.
+ *
+ * See https://jira.zephyrproject.org/browse/ZEP-1595
+ */
+
+#ifdef CONFIG_SYS_POWER_MANAGEMENT
+void _arch_irq_direct_pm(void)
+{
+	if (_kernel.idle) {
+		int32_t idle_val = _kernel.idle;
+
+		_kernel.idle = 0;
+		_sys_power_save_idle_exit(idle_val);
+	}
+}
+#endif
+
+void _arch_isr_direct_header(void)
+{
+	_int_latency_start();
+	_sys_k_event_logger_interrupt();
+	_sys_k_event_logger_exit_sleep();
+
+	/* We're not going to unlock IRQs, but we still need to increment this
+	 * so that _is_in_isr() works
+	 */
+	++_kernel.nested;
+}
+
+void _arch_isr_direct_footer(int swap)
+{
+	_irq_controller_eoi();
+	_int_latency_stop();
+	--_kernel.nested;
+
+	/* Call swap if all the following is true:
+	 *
+	 * 1) swap argument was enabled to this function
+	 * 2) We are not in a nested interrupt
+	 * 3) Current thread is preemptible
+	 * 4) Next thread to run in the ready queue is not this thread
+	 */
+	if (swap && !_kernel.nested &&
+	    _current->base.preempt < _NON_PREEMPT_THRESHOLD &&
+	    _kernel.ready_q.cache != _current) {
+		unsigned int flags;
+
+		/* Fetch EFLAGS argument to _Swap() */
+		__asm__ volatile (
+			"pushfl\n\t"
+			"popl %0\n\t"
+			: "=g" (flags)
+			:
+			: "memory"
+			);
+		_Swap(flags);
+	}
+}
+

include/arch/x86/arch.h

@@ -39,6 +39,14 @@ extern "C" {
 
 #ifndef _ASMLANGUAGE
 
+#ifdef CONFIG_INT_LATENCY_BENCHMARK
+void _int_latency_start(void);
+void _int_latency_stop(void);
+#else
+#define _int_latency_start()  do { } while (0)
+#define _int_latency_stop()   do { } while (0)
+#endif
+
 /* interrupt/exception/error related definitions */
 
 /**
@@ -108,7 +116,8 @@ typedef struct s_isrList {
  */
 
 #define NANO_CPU_INT_REGISTER(r, n, p, v, d) \
-	 ISR_LIST __attribute__((section(".intList"))) MK_ISR_NAME(r) = \
+	 static ISR_LIST __attribute__((section(".intList"))) \
+			 __attribute__((used)) MK_ISR_NAME(r) = \
 			{&r, n, p, v, d}
 
 
@@ -194,6 +203,20 @@ typedef struct s_isrList {
 	_IRQ_TO_INTERRUPT_VECTOR(irq_p); \
 })
 
+/** Configure a 'direct' static interrupt
+ *
+ * All arguments must be computable by the compiler at build time
+ *
+ */
+#define _ARCH_IRQ_DIRECT_CONNECT(irq_p, priority_p, isr_p, flags_p) \
+({ \
+	NANO_CPU_INT_REGISTER(isr_p, irq_p, priority_p, -1, 0); \
+	_irq_controller_irq_config(_IRQ_TO_INTERRUPT_VECTOR(irq_p), (irq_p), \
+				   (flags_p)); \
+	_IRQ_TO_INTERRUPT_VECTOR(irq_p); \
+})
+
+
 #ifdef CONFIG_X86_FIXED_IRQ_MAPPING
 /* Fixed vector-to-irq association mapping.
  * No need for the table at all.
@@ -210,6 +233,31 @@ extern unsigned char _irq_to_interrupt_vector[];
 			((unsigned int) _irq_to_interrupt_vector[irq])
 #endif
 
+#ifdef CONFIG_SYS_POWER_MANAGEMENT
+extern void _arch_irq_direct_pm(void);
+#define _ARCH_ISR_DIRECT_PM() _arch_irq_direct_pm()
+#else
+#define _ARCH_ISR_DIRECT_PM() do { } while (0)
+#endif
+
+#define _ARCH_ISR_DIRECT_HEADER() _arch_isr_direct_header()
+#define _ARCH_ISR_DIRECT_FOOTER(swap) _arch_isr_direct_footer(swap)
+
+/* FIXME prefer these inline, but see ZEP-1595 */
+extern void _arch_isr_direct_header(void);
+extern void _arch_isr_direct_footer(int maybe_swap);
+
+#define _ARCH_ISR_DIRECT_DECLARE(name) \
+	static inline int name##_body(void); \
+	__attribute__ ((interrupt)) void name(void *stack_frame) \
+	{ \
+		ARG_UNUSED(stack_frame); \
+		int check_reschedule; \
+		ISR_DIRECT_HEADER(); \
+		check_reschedule = name##_body(); \
+		ISR_DIRECT_FOOTER(check_reschedule); \
+	} \
+	static inline int name##_body(void)
 
 /**
  * @brief Nanokernel Exception Stack Frame
@@ -294,14 +342,6 @@ typedef struct nanoIsf {
 
 #ifndef _ASMLANGUAGE
 
-#ifdef CONFIG_INT_LATENCY_BENCHMARK
-void _int_latency_start(void);
-void _int_latency_stop(void);
-#else
-#define _int_latency_start()  do { } while (0)
-#define _int_latency_stop()   do { } while (0)
-#endif
-
 /**
  * @brief Disable all interrupts on the CPU (inline)
  *

include/arch/x86/irq_controller.h

@@ -84,6 +84,11 @@ static inline int _irq_controller_isr_vector_get(void)
 }
 
 
+static inline void _irq_controller_eoi(void)
+{
+	__irq_controller_eoi();
+}
+
 #else /* _ASMLANGUAGE */
 
 /**
@@ -98,8 +103,8 @@ static inline int _irq_controller_isr_vector_get(void)
  * saved registers eax, edx, ecx for their own purposes with impunity but
  * need to preserve all callee-saved registers.
  */
-.macro _irq_controller_eoi
+.macro _irq_controller_eoi_macro
-	__irq_controller_eoi
+	__irq_controller_eoi_macro
 .endm
 
 #endif /* _ASMLANGUAGE */

include/drivers/mvic.h

@@ -57,9 +57,14 @@ void __irq_controller_irq_config(unsigned int vector, unsigned int irq,
 
 int __irq_controller_isr_vector_get(void);
 
+static inline void __irq_controller_eoi(void)
+{
+	*(volatile int *)(MVIC_EOI) = 0;
+}
+
 #else /* _ASMLANGUAGE */
 
-.macro __irq_controller_eoi
+.macro __irq_controller_eoi_macro
 	xorl %eax, %eax			/* zeroes eax */
 	movl %eax, MVIC_EOI		/* tell MVIC the IRQ is handled */
 .endm

include/drivers/sysapic.h

@@ -27,14 +27,23 @@ void __irq_controller_irq_config(unsigned int vector, unsigned int irq,
 
 int __irq_controller_isr_vector_get(void);
 
+static inline void __irq_controller_eoi(void)
+{
+#if CONFIG_EOI_FORWARDING_BUG
+	_lakemont_eoi();
+#else
+	*(volatile int *)(CONFIG_LOAPIC_BASE_ADDRESS + LOAPIC_EOI) = 0;
+#endif
+}
+
 #else /* _ASMLANGUAGE */
 
 #if CONFIG_EOI_FORWARDING_BUG
-.macro __irq_controller_eoi
+.macro __irq_controller_eoi_macro
 	call	_lakemont_eoi
 .endm
 #else
-.macro __irq_controller_eoi
+.macro __irq_controller_eoi_macro
 	xorl %eax, %eax			/* zeroes eax */
 	loapic_eoi_reg = (CONFIG_LOAPIC_BASE_ADDRESS + LOAPIC_EOI)
 	movl %eax, loapic_eoi_reg	/* tell LOAPIC the IRQ is handled */