drivers/interrupt_controller: Adding QI support in VT-D

Looks like it's mandatory to invalidate the Interrupt Entry Cache in
VT-D and the only way to do so is to enable Queued Interface.

Signed-off-by: Tomasz Bursztyka <tomasz.bursztyka@linux.intel.com>

drivers/interrupt_controller/intc_intel_vtd.c

@@ -26,6 +26,11 @@
 
 #include "intc_intel_vtd.h"
 
+static inline void vtd_pause_cpu(void)
+{
+	__asm__ volatile("pause" ::: "memory");
+}
+
 static void vtd_write_reg32(const struct device *dev,
 			    uint16_t reg, uint32_t value)
 {
@@ -69,6 +74,94 @@ static void vtd_send_cmd(const struct device *dev,
 	}
 }
 
+static void vtd_qi_init(const struct device *dev)
+{
+	struct vtd_ictl_data *data = dev->data;
+	uint64_t value;
+
+	vtd_write_reg64(dev, VTD_IQT_REG, 0);
+	data->qi_tail = 0;
+
+	value = VTD_IQA_REG_GEN_CONTENT((uintptr_t)data->qi,
+					VTD_IQA_WIDTH_128_BIT, QI_SIZE);
+	vtd_write_reg64(dev, VTD_IQA_REG, value);
+
+	vtd_send_cmd(dev, VTD_GCMD_QIE, VTD_GSTS_QIES);
+}
+
+static inline void vtd_qi_tail_inc(const struct device *dev)
+{
+	struct vtd_ictl_data *data = dev->data;
+
+	data->qi_tail += sizeof(struct qi_descriptor);
+	data->qi_tail %= (QI_NUM * sizeof(struct qi_descriptor));
+}
+
+static int vtd_qi_send(const struct device *dev,
+		       struct qi_descriptor *descriptor)
+{
+	struct vtd_ictl_data *data = dev->data;
+	union qi_wait_descriptor wait_desc = { 0 };
+	struct qi_descriptor *desc;
+	uint32_t wait_status;
+
+	desc = (struct qi_descriptor *)((uintptr_t)data->qi + data->qi_tail);
+
+	desc->low = descriptor->low;
+	desc->high = descriptor->high;
+
+	vtd_qi_tail_inc(dev);
+
+	desc++;
+
+	wait_status = QI_WAIT_STATUS_INCOMPLETE;
+
+	wait_desc.wait.type = QI_TYPE_WAIT;
+	wait_desc.wait.status_write = 1;
+	wait_desc.wait.status_data = QI_WAIT_STATUS_COMPLETE;
+	wait_desc.wait.address = ((uintptr_t)&wait_status) >> 2;
+
+	desc->low = wait_desc.desc.low;
+	desc->high = wait_desc.desc.high;
+
+	vtd_qi_tail_inc(dev);
+
+	vtd_write_reg64(dev, VTD_IQT_REG, data->qi_tail);
+
+	while (wait_status != QI_WAIT_STATUS_COMPLETE) {
+		if (vtd_read_reg32(dev, VTD_FSTS_REG) & VTD_FSTS_IQE) {
+			return -EIO;
+		}
+
+		vtd_pause_cpu();
+	}
+
+	return 0;
+}
+
+static int vtd_global_iec_invalidate(const struct device *dev)
+{
+	union qi_iec_descriptor iec_desc = { 0 };
+
+	iec_desc.iec.type = QI_TYPE_IEC;
+	iec_desc.iec.granularity = 0; /* Global Invalidation requested */
+
+	return vtd_qi_send(dev, &iec_desc.desc);
+}
+
+static int vtd_index_iec_invalidate(const struct device *dev, uint8_t irte_idx)
+{
+	union qi_iec_descriptor iec_desc = { 0 };
+
+	iec_desc.iec.type = QI_TYPE_IEC;
+	iec_desc.iec.granularity = 1; /* Index based invalidation requested */
+
+	iec_desc.iec.interrupt_index = irte_idx;
+	iec_desc.iec.index_mask = 0;
+
+	return vtd_qi_send(dev, &iec_desc.desc);
+}
+
 static void fault_status_description(uint32_t status)
 {
 	if (status & VTD_FSTS_PFO) {
@@ -233,6 +326,8 @@ static int vtd_ictl_remap(const struct device *dev,
 	data->irte[irte_idx].low = irte.low;
 	data->irte[irte_idx].high = irte.high;
 
+	vtd_index_iec_invalidate(dev, irte_idx);
+
 	return 0;
 }
 
@@ -318,11 +413,14 @@ static int vtd_ictl_init(const struct device *dev)
 	unsigned int key = irq_lock();
 	uint64_t eime = 0;
 	uint64_t value;
+	int ret = 0;
 
 	DEVICE_MMIO_MAP(dev, K_MEM_CACHE_NONE);
 
 	vtd_fault_event_init(dev);
 
+	vtd_qi_init(dev);
+
 	if (IS_ENABLED(CONFIG_X2APIC)) {
 		eime = VTD_IRTA_EIME;
 	}
@@ -332,6 +430,12 @@ static int vtd_ictl_init(const struct device *dev)
 
 	vtd_write_reg64(dev, VTD_IRTA_REG, value);
 
+	if (vtd_global_iec_invalidate(dev) != 0) {
+		printk("Could not perform IEC invalidation\n");
+		ret = -EIO;
+		goto out;
+	}
+
 	if (!IS_ENABLED(CONFIG_X2APIC) &&
 	    IS_ENABLED(CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH)) {
 		vtd_send_cmd(dev, VTD_GCMD_CFI, VTD_GSTS_CFIS);
@@ -343,9 +447,10 @@ static int vtd_ictl_init(const struct device *dev)
 	printk("Intel VT-D up and running (status 0x%x)\n",
 	       vtd_read_reg32(dev, VTD_GSTS_REG));
 
+out:
 	irq_unlock(key);
 
-	return 0;
+	return ret;
 }
 
 static const struct vtd_driver_api vtd_api = {

drivers/interrupt_controller/intc_intel_vtd.h

@@ -52,8 +52,59 @@ struct vtd_irte {
 #define IRTE_NUM 256
 #define IRTA_SIZE 7  /* size = 2^(X+1) where IRTA_SIZE is X 2^8 = 256 */
 
+#define QI_NUM 256 /* Which is the minimal number we can set for the queue */
+#define QI_SIZE 0 /* size = 2^(X+8) where QI_SIZE is X: 2^8 = 256 */
+#define QI_WIDTH 128
+
+struct qi_descriptor {
+	uint64_t low;
+	uint64_t high;
+};
+
+#define QI_TYPE_IEC 0x4UL
+
+union qi_iec_descriptor {
+	struct qi_descriptor desc;
+
+	struct iec_bits {
+		uint64_t type		: 4;
+		uint64_t granularity	: 1;
+		uint64_t _reserved_0	: 4;
+		uint64_t zero		: 3;
+		uint64_t _reserved_1	: 15;
+		uint64_t index_mask	: 5;
+		uint64_t interrupt_index: 16;
+		uint64_t _reserved_2	: 16;
+		uint64_t reserved;
+	} iec __packed;
+};
+
+#define QI_TYPE_WAIT 0x5UL
+
+union qi_wait_descriptor {
+	struct qi_descriptor desc;
+
+	struct wait_bits {
+		uint64_t type		: 4;
+		uint64_t interrupt_flag	: 1;
+		uint64_t status_write	: 1;
+		uint64_t fence_flag	: 1;
+		uint64_t page_req_drain	: 1;
+		uint64_t _reserved_0	: 1;
+		uint64_t zero		: 3;
+		uint64_t _reserved_1	: 20;
+		uint64_t status_data	: 32;
+		uint64_t reserved	: 2;
+		uint64_t address	: 62;
+	} wait __packed;
+};
+
+#define QI_WAIT_STATUS_INCOMPLETE 0x0UL
+#define QI_WAIT_STATUS_COMPLETE 0x1UL
+
 struct vtd_ictl_data {
 	struct vtd_irte irte[IRTE_NUM];
+	struct qi_descriptor qi[QI_NUM] __aligned(0x1000);
 	int irqs[IRTE_NUM];
 	int vectors[IRTE_NUM];
 	bool msi[IRTE_NUM];
@@ -61,6 +112,7 @@ struct vtd_ictl_data {
 	unsigned int fault_irq;
 	uintptr_t fault_record_reg;
 	uint16_t fault_record_num;
+	uint16_t qi_tail;
 	uint8_t fault_vector;
 };
 

include/arch/x86/intel_vtd.h

@@ -140,6 +140,7 @@
 #define VTD_FSTS_IQE		BIT(4)
 #define VTD_FSTS_ICE		BIT(5)
 #define VTD_FSTS_ITE		BIT(6)
+
 #define VTD_FSTS_FRI_POS	8
 #define VTD_FSTS_FRI_MASK	(0xF << VTD_FSTS_FRI_POS)
 #define VTD_FSTS_FRI(status)					\
@@ -182,6 +183,21 @@
 #define VTD_FRCD_FI_IR(fault)					\
 	((fault & VTD_FRCD_FI_IR_MASK) >> VTD_FRCD_FI_IR_POS)
 
+/* Invalidation Queue Address register details */
+#define VTD_IQA_SIZE_MASK	0x7
+#define VTD_IQA_WIDTH_128_BIT	0
+#define VTD_IQA_WIDTH_256_BIT	BIT(11)
+#define VTD_IQA_REG_GEN_CONTENT(addr, width, size)			\
+	((uint64_t)0 | (addr) | (width) | (size & VTD_IQA_SIZE_MASK))
+
+/* Invalidation Queue Head register details */
+#define VTD_IQH_QH_POS_128	4
+#define VTD_IQH_QH_MASK		((uint64_t)0xEF << VTD_IQH_QH_POS_128)
+
+/* Invalidation Queue Tail register details */
+#define VTD_IQT_QT_POS_128	4
+#define VTD_IQT_QT_MASK		((uint64_t)0xEF << VTD_IQT_QT_POS_128)
+
 #endif /* _ASMLANGUAGE */
 
 #endif /* ZEPHYR_INCLUDE_ARCH_X86_INTEL_VTD_H */