drivers/interrupt_controller: Add Intel VT-D interrupt remapping driver

Such interrupt remapping controller may be found along with Intel VT-D
hardware. Its base-address is via ACPI, and it enables up to 64K
interrupt indexes.

Signed-off-by: Tomasz Bursztyka <tomasz.bursztyka@linux.intel.com>

drivers/interrupt_controller/CMakeLists.txt

@@ -18,3 +18,4 @@ zephyr_sources_ifdef(CONFIG_VEXRISCV_LITEX_IRQ      intc_vexriscv_litex.c)
 zephyr_sources_ifdef(CONFIG_SWERV_PIC               intc_swerv_pic.c)
 zephyr_sources_ifdef(CONFIG_NPCX_MIWU               intc_miwu.c)
 zephyr_sources_ifdef(CONFIG_LEON_IRQMP              intc_irqmp.c)
+zephyr_sources_ifdef(CONFIG_INTEL_VTD_ICTL          intc_intel_vtd.c)

drivers/interrupt_controller/Kconfig

@@ -64,4 +64,6 @@ source "drivers/interrupt_controller/Kconfig.gic"
 
 source "drivers/interrupt_controller/Kconfig.npcx"
 
+source "drivers/interrupt_controller/Kconfig.intel_vtd"
+
 endmenu

drivers/interrupt_controller/Kconfig.intel_vtd

@@ -0,0 +1,25 @@
+# Intel VT-D interrupt remapping controller configuration
+
+# Copyright (c) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+menuconfig INTEL_VTD_ICTL
+	bool "Intel VT-D interrupt remapping controller"
+	depends on ACPI && X86 && 64BIT
+	select PCIE_MSI_MULTI_VECTOR
+	help
+	  Such interrupt remapping hardware is provided through Intel VT-D
+	  technology. It's being used, currently, only for MSI/MSI-X
+	  multi-vector support. If you have such PCIe device requiring
+	  multi-vector support, you will need to enable this.
+
+if INTEL_VTD_ICTL
+
+config INTEL_VTD_ICTL_INIT_PRIORITY
+	int "Initialization priority"
+	default 0
+	help
+	  This device should be initialized as soon as possible, before any
+	  other device that would require it for MSI/MSI-X multi-vector support.
+
+endif # INTEL_VTD_ICTL

drivers/interrupt_controller/intc_intel_vtd.c

@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2020 Intel Corporation
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#define DT_DRV_COMPAT intel_vt_d
+
+#include <errno.h>
+
+#include <kernel.h>
+#include <arch/cpu.h>
+
+#include <soc.h>
+#include <device.h>
+#include <init.h>
+#include <string.h>
+
+#include <zephyr.h>
+
+#include <arch/x86/intel_vtd.h>
+#include <drivers/interrupt_controller/intel_vtd.h>
+
+#include "intc_intel_vtd.h"
+
+static void vtd_write_reg64(const struct device *dev,
+			    uint16_t reg, uint64_t value)
+{
+	uintptr_t base_address = DEVICE_MMIO_GET(dev);
+
+	sys_write64(value, (base_address + reg));
+}
+
+static uint32_t vtd_read_reg(const struct device *dev, uint16_t reg)
+{
+	uintptr_t base_address = DEVICE_MMIO_GET(dev);
+
+	return sys_read32(base_address + reg);
+}
+
+static void vtd_send_cmd(const struct device *dev,
+			 uint16_t cmd_bit, uint16_t status_bit)
+{
+	uintptr_t base_address = DEVICE_MMIO_GET(dev);
+
+	sys_set_bit((base_address + VTD_GCMD_REG), cmd_bit);
+
+	while (!sys_test_bit((base_address + VTD_GSTS_REG),
+			     status_bit)) {
+		/* Do nothing */
+	}
+
+}
+
+static int vtd_ictl_allocate_entries(const struct device *dev,
+				     uint8_t n_entries)
+{
+	struct vtd_ictl_data *data = dev->data;
+	int irte_idx_start;
+
+	if ((data->irte_num_used + n_entries) > IRTE_NUM) {
+		return -EBUSY;
+	}
+
+	irte_idx_start = data->irte_num_used;
+	data->irte_num_used += n_entries;
+
+	return irte_idx_start;
+}
+
+static uint32_t vtd_ictl_remap_msi(const struct device *dev,
+				   msi_vector_t *vector)
+{
+	return VTD_MSI_MAP(vector->arch.irte);
+}
+
+static int vtd_ictl_remap(const struct device *dev,
+			  msi_vector_t *vector)
+{
+	struct vtd_ictl_data *data = dev->data;
+	uint8_t irte_idx = vector->arch.irte;
+
+	memset(&data->irte[irte_idx], 0, sizeof(struct vtd_irte));
+
+	data->irte[irte_idx].l.vector = vector->arch.vector;
+	data->irte[irte_idx].l.dst_id = arch_curr_cpu()->id;
+	data->irte[irte_idx].l.present = 1;
+
+	return 0;
+}
+
+static int vtd_ictl_init(const struct device *dev)
+{
+	struct vtd_ictl_data *data = dev->data;
+	unsigned int key = irq_lock();
+	uint64_t eime = 0;
+	uint64_t irta;
+
+	DEVICE_MMIO_MAP(dev, K_MEM_CACHE_NONE);
+
+	if (IS_ENABLED(CONFIG_X2APIC)) {
+		eime = VTD_IRTA_EIME;
+	}
+
+	irta = VTD_IRTA_REG_GEN_CONTENT((uintptr_t)data->irte,
+					IRTA_SIZE, eime);
+
+	vtd_write_reg64(dev, VTD_IRTA_REG, irta);
+
+	vtd_send_cmd(dev, VTD_GCMD_SIRTP, VTD_GSTS_SIRTPS);
+	vtd_send_cmd(dev, VTD_GCMD_IRE, VTD_GSTS_IRES);
+
+	printk("Intel VT-D up and running (status 0x%x)\n",
+	       vtd_read_reg(dev, VTD_GSTS_REG));
+
+	irq_unlock(key);
+
+	return 0;
+}
+
+static const struct vtd_driver_api vtd_api = {
+	.allocate_entries = vtd_ictl_allocate_entries,
+	.remap_msi = vtd_ictl_remap_msi,
+	.remap = vtd_ictl_remap,
+};
+
+static struct vtd_ictl_data vtd_ictl_data_0;
+
+static const struct vtd_ictl_cfg vtd_ictl_cfg_0 = {
+	DEVICE_MMIO_ROM_INIT(DT_DRV_INST(0)),
+};
+
+DEVICE_DEFINE(vtd_ictl, DT_INST_LABEL(0),
+	      vtd_ictl_init, device_pm_control_nop,
+	      &vtd_ictl_data_0, &vtd_ictl_cfg_0,
+	      PRE_KERNEL_1, CONFIG_INTEL_VTD_ICTL_INIT_PRIORITY, &vtd_api);

drivers/interrupt_controller/intc_intel_vtd.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2020 Intel Corporation
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef ZEPHYR_DRIVERS_INTERRUPT_CONTROLLER_INTC_INTEL_VTD_H_
+#define ZEPHYR_DRIVERS_INTERRUPT_CONTROLLER_INTC_INTEL_VTD_H_
+
+#define VTD_INT_SHV	BIT(3)
+#define VTD_INT_FORMAT	BIT(4)
+
+/* We don't care about int_idx[15], since the size is fixed to 256,
+ * it's always 0
+ */
+#define VTD_MSI_MAP(int_idx) \
+	((0x0FEE << 20) | (int_idx << 5) | VTD_INT_SHV | VTD_INT_FORMAT)
+
+/* Interrupt Remapping Table Entry (IRTE) for Remapped Interrupts */
+struct vtd_irte {
+	struct {
+		uint64_t present		: 1;
+		uint64_t fpd			: 1;
+		uint64_t dst_mode		: 1;
+		uint64_t redirection_hint	: 1;
+		uint64_t trigger_mode		: 1;
+		uint64_t delivery_mode		: 3;
+		uint64_t available		: 4;
+		uint64_t _reserved_0		: 3;
+		uint64_t irte_mode		: 1;
+		uint64_t vector			: 8;
+		uint64_t _reserved_1		: 8;
+		uint64_t dst_id			: 32;
+	} l;
+
+	struct {
+		uint64_t src_id			: 16;
+		uint64_t src_id_qualifier	: 2;
+		uint64_t src_validation_type	: 2;
+		uint64_t _reserved		: 44;
+	} h;
+} __packed;
+
+/* The table must be 4KB aligned, which is exactly 256 entries.
+ * And since we allow only 256 entries as a maximum: let's align to it.
+ */
+#define IRTE_NUM 256
+#define IRTA_SIZE 7  /* size = 2^(X+1) where IRTA_SIZE is X 2^8 = 256 */
+
+struct vtd_ictl_data {
+	struct vtd_irte irte[IRTE_NUM];
+	int irte_num_used;
+};
+
+struct vtd_ictl_cfg {
+	DEVICE_MMIO_ROM;
+};
+
+#endif /* ZEPHYR_DRIVERS_INTERRUPT_CONTROLLER_INTC_INTEL_VTD_H_ */

include/arch/x86/arch.h

@@ -32,6 +32,10 @@ extern "C" {
 struct x86_msi_vector {
 	unsigned int irq;
 	uint8_t vector;
+#ifdef CONFIG_INTEL_VTD_ICTL
+	bool remap;
+	uint8_t irte;
+#endif
 };
 
 typedef struct x86_msi_vector arch_msi_vector_t;

include/arch/x86/intel_vtd.h

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2020 Intel Corporation
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef ZEPHYR_INCLUDE_ARCH_X86_INTEL_VTD_H
+#define ZEPHYR_INCLUDE_ARCH_X86_INTEL_VTD_H
+
+#ifndef _ASMLANGUAGE
+
+/*************\
+ * Registers *
+\*************/
+
+#define VTD_VER_REG		0x000 /* Version */
+#define VTD_CAP_REG		0x008 /* Capability */
+#define VTD_ECAP_REG		0x010 /* Extended Capability */
+#define VTD_GCMD_REG		0x018 /* Global Command */
+#define VTD_GSTS_REG		0x01C /* Global Status */
+#define VTD_RTADDR_REG		0x020 /* Root Table Address */
+#define VTD_CCMD_REG		0x028 /* Context Command */
+#define VTD_FSTS_REG		0x034 /* Fault Status */
+#define VTD_FECTL_REG		0x038 /* Fault Event Control Register*/
+#define VTD_FEDATA_REG		0x03C /* Fault Event Data */
+#define VTD_FEADDR_REG		0x040 /* Fault Event Address */
+#define VTD_FEUADDR_REG		0x044 /* Fault Event Upper Address */
+#define VTD_AFLOG_REG		0x058 /* Advanced Fault Log */
+#define VTD_PMEN_REG		0x064 /* Protected Memory Enable */
+#define VTD_PLMBASE_REG		0x068 /* Protected Low Memory Base */
+#define VTD_PLMLIMIT_REG	0x06C /* Protected Low Memory Limit */
+#define VTD_PHMBASE_REG		0x070 /* Protected High Memory Base */
+#define VTD_PHMLIMIT_REG	0x078 /* Protected High Memory Limit */
+#define VTD_IQH_REG		0x080 /* Invalidation Queue Head */
+#define VTD_IQT_REG		0x088 /* Invalidation Queue Tail */
+#define VTD_IQA_REG		0x090 /* Invalidation Queue Address */
+#define VTD_ICS_REG		0x09C /* Invalidation Completion Status */
+#define VTD_IECTL_REG		0x0A0 /* Invalidation Completion Event Control */
+#define VTD_IEDATA_REG		0x0A4 /* Invalidation Completion Event Data */
+#define VTD_IEADDR_REG		0x0A8 /* Invalidation Completion Event Address */
+#define VTD_IEUADDR_REG		0x0AC /* Invalidation Completion Event Upper Address */
+#define VTD_IQERCD_REG		0x0B0 /* Invalidation Queue Error Record */
+#define VTD_IRTA_REG		0x0B8 /* Interrupt Remapping Table Address */
+#define VTD_PQH_REG		0x0C0 /* Page Request Queue Head */
+#define VTD_PQT_REG		0x0C8 /* Page Request Queue Tail */
+#define VTD_PQA_REG		0x0D0 /* Page Request Queue Address */
+#define VTD_PRS_REG		0x0DC /* Page Request Status */
+#define VTD_PECTL_REG		0x0E0 /* Page Request Event Control */
+#define VTD_PEDATA_REG		0x0E4 /* Page Request Event Data */
+#define VTD_PEADDR_REG		0x0E8 /* Page Request Event Address */
+#define VTD_PEUADDR_REG		0x0EC /* Page Request Event Upper Address */
+#define VTD_MTRRCAP_REG		0x100 /* MTRR Capability */
+#define VTD_MTRRDEF_REG		0x108 /* MTRR Default Type */
+#define VTD_MTRR_FIX64K_00000_REG 0x120 /* Fixed-range MTRR for 64K_00000 */
+#define VTD_MTRR_FIX16K_80000_REG 0x128 /* Fixed-range MTRR for 16K_80000 */
+#define VTD_MTRR_FIX16K_A0000_REG 0x130 /* Fixed-range MTRR for 16K_A0000 */
+#define VTD_MTRR_FIX4K_C0000_REG 0x138 /* Fixed-range MTRR for 4K_C0000 */
+#define VTD_MTRR_FIX4K_C8000_REG 0x140 /* Fixed-range MTRR for 4K_C8000 */
+#define VTD_MTRR_FIX4K_D0000_REG 0x148 /* Fixed-range MTRR for 4K_D0000 */
+#define VTD_MTRR_FIX4K_D8000_REG 0x150 /* Fixed-range MTRR for 4K_D8000 */
+#define VTD_MTRR_FIX4K_E0000_REG 0x158 /* Fixed-range MTRR for 4K_E0000 */
+#define VTD_MTRR_FIX4K_E8000_REG 0x160 /* Fixed-range MTRR for 4K_E8000 */
+#define VTD_MTRR_FIX4K_F0000_REG 0x168 /* Fixed-range MTRR for 4K_F0000 */
+#define VTD_MTRR_FIX4K_F8000_REG 0x170 /* Fixed-range MTRR for 4K_F8000 */
+#define VTD_MTRR_PHYSBASE0_REG	0x180 /* Variable-range MTRR Base0 */
+#define VTD_MTRR_PHYSMASK0_REG	0x188 /* Variable-range MTRR Mask0 */
+#define VTD_MTRR_PHYSBASE1_REG	0x190 /* Variable-range MTRR Base1 */
+#define VTD_MTRR_PHYSMASK1_REG	0x198 /* Variable-range MTRR Mask1 */
+#define VTD_MTRR_PHYSBASE2_REG	0x1A0 /* Variable-range MTRR Base2 */
+#define VTD_MTRR_PHYSMASK2_REG	0x1A8 /* Variable-range MTRR Mask2 */
+#define VTD_MTRR_PHYSBASE3_REG	0x1B0 /* Variable-range MTRR Base3 */
+#define VTD_MTRR_PHYSMASK3_REG	0x1B8 /* Variable-range MTRR Mask3 */
+#define VTD_MTRR_PHYSBASE4_REG	0x1C0 /* Variable-range MTRR Base4 */
+#define VTD_MTRR_PHYSMASK4_REG	0x1C8 /* Variable-range MTRR Mask4 */
+#define VTD_MTRR_PHYSBASE5_REG	0x1D0 /* Variable-range MTRR Base5 */
+#define VTD_MTRR_PHYSMASK5_REG	0x1D8 /* Variable-range MTRR Mask5 */
+#define VTD_MTRR_PHYSBASE6_REG	0x1E0 /* Variable-range MTRR Base6 */
+#define VTD_MTRR_PHYSMASK6_REG	0x1E8 /* Variable-range MTRR Mask6 */
+#define VTD_MTRR_PHYSBASE7_REG	0x1F0 /* Variable-range MTRR Base7 */
+#define VTD_MTRR_PHYSMASK7_REG	0x1F8 /* Variable-range MTRR Mask7 */
+#define VTD_MTRR_PHYSBASE8_REG	0x200 /* Variable-range MTRR Base8 */
+#define VTD_MTRR_PHYSMASK8_REG	0x208 /* Variable-range MTRR Mask8 */
+#define VTD_MTRR_PHYSBASE9_REG	0x210 /* Variable-range MTRR Base9 */
+#define VTD_MTRR_PHYSMASK9_REG	0x218 /* Variable-range MTRR Mask9 */
+#define VTD_VCCAP_REG		0xE00 /* Virtual Command Capability */
+#define VTD_VCMD		0xE10 /* Virtual Command */
+#define VTD_VCRSP		0xE20 /* Virtual Command Response */
+
+/* Global Command Register details */
+#define VTD_GCMD_CFI		23
+#define VTD_GCMD_SIRTP		24
+#define VTD_GCMD_IRE		25
+#define VTD_GCMD_QIE		26
+#define VTD_GCMD_WBF		27
+#define VTD_GCMD_EAFL		28
+#define VTD_GCMD_SFL		29
+#define VTD_GCMD_SRTP		30
+#define VTD_GCMD_TE		31
+
+/* Global Status Register details */
+#define VTD_GSTS_CFIS		23
+#define VTD_GSTS_SIRTPS		24
+#define VTD_GSTS_IRES		25
+#define VTD_GSTS_QIES		26
+#define VTD_GSTS_WBFS		27
+#define VTD_GSTS_EAFLS		28
+#define VTD_GSTS_SFLS		29
+#define VTD_GSTS_SRTPS		30
+#define VTD_GSTS_TES		31
+
+/* Interrupt Remapping Table Address Register details */
+#define VTD_IRTA_SIZE_MASK	0x00000000000000FF
+#define VTD_IRTA_EIME		11
+#define VTD_IRTA_ADDR_SHIFT	12
+
+#define VTD_IRTA_REG_GEN_CONTENT(addr, size, mode) \
+	(0 |					   \
+	 (addr << VTD_IRTA_ADDR_SHIFT) |	   \
+	 (mode) | (size & VTD_IRTA_SIZE_MASK))
+
+#endif /* _ASMLANGUAGE */
+
+#endif /* ZEPHYR_INCLUDE_ARCH_X86_INTEL_VTD_H */

include/drivers/interrupt_controller/intel_vtd.h

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2020 Intel Corporation
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef ZEPHYR_INCLUDE_DRIVERS_INTEL_VTD_H_
+#define ZEPHYR_INCLUDE_DRIVERS_INTEL_VTD_H_
+
+#include <drivers/pcie/msi.h>
+
+typedef int (*vtd_alloc_entries_f)(const struct device *dev,
+				   uint8_t n_entries);
+
+typedef uint32_t (*vtd_remap_msi_f)(const struct device *dev,
+				    msi_vector_t *vector);
+
+typedef int (*vtd_remap_f)(const struct device *dev,
+			   msi_vector_t *vector);
+
+struct vtd_driver_api {
+	vtd_alloc_entries_f allocate_entries;
+	vtd_remap_msi_f remap_msi;
+	vtd_remap_f remap;
+};
+
+/**
+ * @brief Allocate contiguous IRTEs
+ *
+ * @param dev Pointer to the device structure for the driver instance
+ * @param n_entries How many IRTE to allocate
+ *
+ * Note: It will try to allocate all, or it will fail.
+ *
+ * @return The first allocated IRTE index, or -EBUSY on failure
+ */
+static inline int vtd_allocate_entries(const struct device *dev,
+				       uint8_t n_entries)
+{
+	const struct vtd_driver_api *api =
+		(const struct vtd_driver_api *)dev->api;
+
+	return api->allocate_entries(dev, n_entries);
+}
+
+/**
+ * @brief Generate the MSI Message Address data for the given vector
+ *
+ * @param dev Pointer to the device structure for the driver instance
+ * @param vector A valid allocated MSI vector
+ *
+ * @return The MSI Message Address value
+ */
+static inline uint32_t vtd_remap_msi(const struct device *dev,
+				     msi_vector_t *vector)
+{
+	const struct vtd_driver_api *api =
+		(const struct vtd_driver_api *)dev->api;
+
+	return api->remap_msi(dev, vector);
+}
+
+/**
+ * @brief Remap the given vector
+ *
+ * @param dev Pointer to the device structure for the driver instance
+ * @param vector A valid allocated MSI vector
+ *
+ * @return 0 on success, a negative errno otherwise
+ */
+static inline int vtd_remap(const struct device *dev,
+			    msi_vector_t *vector)
+{
+	const struct vtd_driver_api *api =
+		(const struct vtd_driver_api *)dev->api;
+
+	return api->remap(dev, vector);
+}
+
+
+#endif /* ZEPHYR_INCLUDE_DRIVERS_INTEL_VTD_H_ */