From 2c904b379bf7e1f8301be37520e86d843672ff1e Mon Sep 17 00:00:00 2001 From: Tomasz Bursztyka Date: Tue, 12 Jan 2021 14:03:04 +0100 Subject: [PATCH] drivers/interrupt_controller: Adding QI support in VT-D Looks like it's mandatory to invalidate the Interrupt Entry Cache in VT-D and the only way to do so is to enable Queued Interface. Signed-off-by: Tomasz Bursztyka --- drivers/interrupt_controller/intc_intel_vtd.c | 107 +++++++++++++++++- drivers/interrupt_controller/intc_intel_vtd.h | 52 +++++++++ include/arch/x86/intel_vtd.h | 16 +++ 3 files changed, 174 insertions(+), 1 deletion(-) diff --git a/drivers/interrupt_controller/intc_intel_vtd.c b/drivers/interrupt_controller/intc_intel_vtd.c index 3cce54dab71..61303f9178f 100644 --- a/drivers/interrupt_controller/intc_intel_vtd.c +++ b/drivers/interrupt_controller/intc_intel_vtd.c @@ -26,6 +26,11 @@ #include "intc_intel_vtd.h" +static inline void vtd_pause_cpu(void) +{ + __asm__ volatile("pause" ::: "memory"); +} + static void vtd_write_reg32(const struct device *dev, uint16_t reg, uint32_t value) { @@ -69,6 +74,94 @@ static void vtd_send_cmd(const struct device *dev, } } +static void vtd_qi_init(const struct device *dev) +{ + struct vtd_ictl_data *data = dev->data; + uint64_t value; + + vtd_write_reg64(dev, VTD_IQT_REG, 0); + data->qi_tail = 0; + + value = VTD_IQA_REG_GEN_CONTENT((uintptr_t)data->qi, + VTD_IQA_WIDTH_128_BIT, QI_SIZE); + vtd_write_reg64(dev, VTD_IQA_REG, value); + + vtd_send_cmd(dev, VTD_GCMD_QIE, VTD_GSTS_QIES); +} + +static inline void vtd_qi_tail_inc(const struct device *dev) +{ + struct vtd_ictl_data *data = dev->data; + + data->qi_tail += sizeof(struct qi_descriptor); + data->qi_tail %= (QI_NUM * sizeof(struct qi_descriptor)); +} + +static int vtd_qi_send(const struct device *dev, + struct qi_descriptor *descriptor) +{ + struct vtd_ictl_data *data = dev->data; + union qi_wait_descriptor wait_desc = { 0 }; + struct qi_descriptor *desc; + uint32_t wait_status; + + desc = (struct qi_descriptor *)((uintptr_t)data->qi + data->qi_tail); + + desc->low = descriptor->low; + desc->high = descriptor->high; + + vtd_qi_tail_inc(dev); + + desc++; + + wait_status = QI_WAIT_STATUS_INCOMPLETE; + + wait_desc.wait.type = QI_TYPE_WAIT; + wait_desc.wait.status_write = 1; + wait_desc.wait.status_data = QI_WAIT_STATUS_COMPLETE; + wait_desc.wait.address = ((uintptr_t)&wait_status) >> 2; + + desc->low = wait_desc.desc.low; + desc->high = wait_desc.desc.high; + + vtd_qi_tail_inc(dev); + + vtd_write_reg64(dev, VTD_IQT_REG, data->qi_tail); + + while (wait_status != QI_WAIT_STATUS_COMPLETE) { + if (vtd_read_reg32(dev, VTD_FSTS_REG) & VTD_FSTS_IQE) { + return -EIO; + } + + vtd_pause_cpu(); + } + + return 0; +} + +static int vtd_global_iec_invalidate(const struct device *dev) +{ + union qi_iec_descriptor iec_desc = { 0 }; + + iec_desc.iec.type = QI_TYPE_IEC; + iec_desc.iec.granularity = 0; /* Global Invalidation requested */ + + return vtd_qi_send(dev, &iec_desc.desc); +} + +static int vtd_index_iec_invalidate(const struct device *dev, uint8_t irte_idx) +{ + union qi_iec_descriptor iec_desc = { 0 }; + + iec_desc.iec.type = QI_TYPE_IEC; + iec_desc.iec.granularity = 1; /* Index based invalidation requested */ + + iec_desc.iec.interrupt_index = irte_idx; + iec_desc.iec.index_mask = 0; + + return vtd_qi_send(dev, &iec_desc.desc); +} + static void fault_status_description(uint32_t status) { if (status & VTD_FSTS_PFO) { @@ -233,6 +326,8 @@ static int vtd_ictl_remap(const struct device *dev, data->irte[irte_idx].low = irte.low; data->irte[irte_idx].high = irte.high; + vtd_index_iec_invalidate(dev, irte_idx); + return 0; } @@ -318,11 +413,14 @@ static int vtd_ictl_init(const struct device *dev) unsigned int key = irq_lock(); uint64_t eime = 0; uint64_t value; + int ret = 0; DEVICE_MMIO_MAP(dev, K_MEM_CACHE_NONE); vtd_fault_event_init(dev); + vtd_qi_init(dev); + if (IS_ENABLED(CONFIG_X2APIC)) { eime = VTD_IRTA_EIME; } @@ -332,6 +430,12 @@ static int vtd_ictl_init(const struct device *dev) vtd_write_reg64(dev, VTD_IRTA_REG, value); + if (vtd_global_iec_invalidate(dev) != 0) { + printk("Could not perform IEC invalidation\n"); + ret = -EIO; + goto out; + } + if (!IS_ENABLED(CONFIG_X2APIC) && IS_ENABLED(CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH)) { vtd_send_cmd(dev, VTD_GCMD_CFI, VTD_GSTS_CFIS); @@ -343,9 +447,10 @@ static int vtd_ictl_init(const struct device *dev) printk("Intel VT-D up and running (status 0x%x)\n", vtd_read_reg32(dev, VTD_GSTS_REG)); +out: irq_unlock(key); - return 0; + return ret; } static const struct vtd_driver_api vtd_api = { diff --git a/drivers/interrupt_controller/intc_intel_vtd.h b/drivers/interrupt_controller/intc_intel_vtd.h index 2789dda921d..22caea61613 100644 --- a/drivers/interrupt_controller/intc_intel_vtd.h +++ b/drivers/interrupt_controller/intc_intel_vtd.h @@ -52,8 +52,59 @@ struct vtd_irte { #define IRTE_NUM 256 #define IRTA_SIZE 7 /* size = 2^(X+1) where IRTA_SIZE is X 2^8 = 256 */ +#define QI_NUM 256 /* Which is the minimal number we can set for the queue */ +#define QI_SIZE 0 /* size = 2^(X+8) where QI_SIZE is X: 2^8 = 256 */ +#define QI_WIDTH 128 + +struct qi_descriptor { + uint64_t low; + uint64_t high; +}; + +#define QI_TYPE_IEC 0x4UL + +union qi_iec_descriptor { + struct qi_descriptor desc; + + struct iec_bits { + uint64_t type : 4; + uint64_t granularity : 1; + uint64_t _reserved_0 : 4; + uint64_t zero : 3; + uint64_t _reserved_1 : 15; + uint64_t index_mask : 5; + uint64_t interrupt_index: 16; + uint64_t _reserved_2 : 16; + uint64_t reserved; + } iec __packed; +}; + +#define QI_TYPE_WAIT 0x5UL + +union qi_wait_descriptor { + struct qi_descriptor desc; + + struct wait_bits { + uint64_t type : 4; + uint64_t interrupt_flag : 1; + uint64_t status_write : 1; + uint64_t fence_flag : 1; + uint64_t page_req_drain : 1; + uint64_t _reserved_0 : 1; + uint64_t zero : 3; + uint64_t _reserved_1 : 20; + uint64_t status_data : 32; + uint64_t reserved : 2; + uint64_t address : 62; + } wait __packed; +}; + +#define QI_WAIT_STATUS_INCOMPLETE 0x0UL +#define QI_WAIT_STATUS_COMPLETE 0x1UL + struct vtd_ictl_data { struct vtd_irte irte[IRTE_NUM]; + struct qi_descriptor qi[QI_NUM] __aligned(0x1000); int irqs[IRTE_NUM]; int vectors[IRTE_NUM]; bool msi[IRTE_NUM]; @@ -61,6 +112,7 @@ struct vtd_ictl_data { unsigned int fault_irq; uintptr_t fault_record_reg; uint16_t fault_record_num; + uint16_t qi_tail; uint8_t fault_vector; }; diff --git a/include/arch/x86/intel_vtd.h b/include/arch/x86/intel_vtd.h index 2f6ebbd1b67..2fe005cb327 100644 --- a/include/arch/x86/intel_vtd.h +++ b/include/arch/x86/intel_vtd.h @@ -140,6 +140,7 @@ #define VTD_FSTS_IQE BIT(4) #define VTD_FSTS_ICE BIT(5) #define VTD_FSTS_ITE BIT(6) + #define VTD_FSTS_FRI_POS 8 #define VTD_FSTS_FRI_MASK (0xF << VTD_FSTS_FRI_POS) #define VTD_FSTS_FRI(status) \ @@ -182,6 +183,21 @@ #define VTD_FRCD_FI_IR(fault) \ ((fault & VTD_FRCD_FI_IR_MASK) >> VTD_FRCD_FI_IR_POS) +/* Invalidation Queue Address register details */ +#define VTD_IQA_SIZE_MASK 0x7 +#define VTD_IQA_WIDTH_128_BIT 0 +#define VTD_IQA_WIDTH_256_BIT BIT(11) +#define VTD_IQA_REG_GEN_CONTENT(addr, width, size) \ + ((uint64_t)0 | (addr) | (width) | (size & VTD_IQA_SIZE_MASK)) + +/* Invalidation Queue Head register details */ +#define VTD_IQH_QH_POS_128 4 +#define VTD_IQH_QH_MASK ((uint64_t)0xEF << VTD_IQH_QH_POS_128) + +/* Invalidation Queue Tail register details */ +#define VTD_IQT_QT_POS_128 4 +#define VTD_IQT_QT_MASK ((uint64_t)0xEF << VTD_IQT_QT_POS_128) + #endif /* _ASMLANGUAGE */ #endif /* ZEPHYR_INCLUDE_ARCH_X86_INTEL_VTD_H */