interrupt_controller: gicv3: add Interrupt Translation Service support
This implements support for the optional Interrupt Translation Service (ITS) module of the GICv3 Interrupt Controller. The current implementation is designed for MSI/MSI-X interrupt delivery in mind. The gicv3 driver calls each ITS INVALL command when LPI interrupts are enabled/disabled. A simple atomic integer is used to allocate unique LPI INTIDs to ITS users. CPUs numbers are directly mapped as ICIDs into the Collections Table. As a limitation it doesn't support indirect Device table to simplify implementation but may use a large amount of memory. INV, DISCARD, MOVI and MOVALL commands are not implemented. Signed-off-by: Neil Armstrong <narmstrong@baylibre.com>
This commit is contained in:
parent
e819bd8f34
commit
a737999b5b
5 changed files with 781 additions and 7 deletions
|
@ -9,6 +9,7 @@ zephyr_library_sources_ifdef(CONFIG_EXTI_STM32 intc_exti_stm32.c)
|
|||
zephyr_library_sources_ifdef(CONFIG_GIC_V1 intc_gic.c)
|
||||
zephyr_library_sources_ifdef(CONFIG_GIC_V2 intc_gic.c)
|
||||
zephyr_library_sources_ifdef(CONFIG_GIC_V3 intc_gicv3.c)
|
||||
zephyr_library_sources_ifdef(CONFIG_GIC_V3_ITS intc_gicv3_its.c)
|
||||
zephyr_library_sources_ifdef(CONFIG_INTEL_VTD_ICTL intc_intel_vtd.c)
|
||||
zephyr_library_sources_ifdef(CONFIG_IOAPIC intc_ioapic.c)
|
||||
zephyr_library_sources_ifdef(CONFIG_ITE_IT8XXX2_INTC intc_ite_it8xxx2.c)
|
||||
|
|
|
@ -43,4 +43,14 @@ config GIC_SINGLE_SECURITY_STATE
|
|||
Some ARM Cortex-family processors only supports single security
|
||||
state.
|
||||
|
||||
config GIC_V3_ITS
|
||||
bool "GIC v3 Interrupt Translation Service"
|
||||
depends on GIC_V3
|
||||
# ITS generates Non-secure Group 1 LPI interrupts, requiring EL1NS
|
||||
select ARMV8_A_NS
|
||||
select DYNAMIC_INTERRUPTS
|
||||
help
|
||||
Support for the optional Interrupt Translation Service used to translate
|
||||
hardware interrupt from PCIe MSI messages for example.
|
||||
|
||||
endif # CPU_CORTEX
|
||||
|
|
|
@ -34,6 +34,8 @@ mem_addr_t gic_rdists[CONFIG_MP_NUM_CPUS];
|
|||
|
||||
#ifdef CONFIG_GIC_V3_ITS
|
||||
static uintptr_t lpi_prop_table;
|
||||
|
||||
atomic_t nlpi_intid = ATOMIC_INIT(8192);
|
||||
#endif
|
||||
|
||||
static inline mem_addr_t gic_get_rdist(void)
|
||||
|
@ -76,6 +78,8 @@ static void arm_gic_lpi_setup(unsigned int intid, bool enable)
|
|||
}
|
||||
|
||||
dsb();
|
||||
|
||||
its_rdist_invall();
|
||||
}
|
||||
|
||||
static void arm_gic_lpi_set_priority(unsigned int intid, unsigned int prio)
|
||||
|
@ -86,6 +90,8 @@ static void arm_gic_lpi_set_priority(unsigned int intid, unsigned int prio)
|
|||
*cfg |= prio & 0xfc;
|
||||
|
||||
dsb();
|
||||
|
||||
its_rdist_invall();
|
||||
}
|
||||
|
||||
static bool arm_gic_lpi_is_enabled(unsigned int intid)
|
||||
|
@ -483,5 +489,10 @@ SYS_INIT(arm_gic_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_DEFAULT);
|
|||
void arm_gic_secondary_init(void)
|
||||
{
|
||||
__arm_gic_init();
|
||||
|
||||
#ifdef CONFIG_GIC_V3_ITS
|
||||
/* Map this CPU Redistributor in all the ITS Collection tables */
|
||||
its_rdist_map();
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
|
576
drivers/interrupt_controller/intc_gicv3_its.c
Normal file
576
drivers/interrupt_controller/intc_gicv3_its.c
Normal file
|
@ -0,0 +1,576 @@
|
|||
/*
|
||||
* Copyright 2021 BayLibre, SAS
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <logging/log.h>
|
||||
LOG_MODULE_REGISTER(intc_gicv3_its, LOG_LEVEL_ERR);
|
||||
|
||||
#include <kernel.h>
|
||||
#include <device.h>
|
||||
#include <drivers/interrupt_controller/gicv3_its.h>
|
||||
|
||||
#include "intc_gic_common_priv.h"
|
||||
#include "intc_gicv3_priv.h"
|
||||
|
||||
#define DT_DRV_COMPAT arm_gic_v3_its
|
||||
|
||||
/*
|
||||
* Current ITS implementation only handle GICv3 ITS physical interruption generation
|
||||
* Implementation is designed for the PCIe MSI/MSI-X use-case in mind.
|
||||
*/
|
||||
|
||||
#define GITS_BASER_NR_REGS 8
|
||||
|
||||
/* convenient access to all redistributors base address */
|
||||
extern mem_addr_t gic_rdists[CONFIG_MP_NUM_CPUS];
|
||||
|
||||
#define SIZE_256 256
|
||||
#define SIZE_4K KB(4)
|
||||
#define SIZE_16K KB(16)
|
||||
#define SIZE_64K KB(64)
|
||||
|
||||
struct its_cmd_block {
|
||||
uint64_t raw_cmd[4];
|
||||
};
|
||||
|
||||
#define ITS_CMD_QUEUE_SIZE SIZE_64K
|
||||
#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SIZE / sizeof(struct its_cmd_block))
|
||||
|
||||
struct gicv3_its_data {
|
||||
mm_reg_t base;
|
||||
struct its_cmd_block *cmd_base;
|
||||
struct its_cmd_block *cmd_write;
|
||||
};
|
||||
|
||||
struct gicv3_its_config {
|
||||
uintptr_t base_addr;
|
||||
size_t base_size;
|
||||
struct its_cmd_block *cmd_queue;
|
||||
size_t cmd_queue_size;
|
||||
};
|
||||
|
||||
/* wait 500ms & wakeup every millisecond */
|
||||
#define WAIT_QUIESCENT 500
|
||||
|
||||
static int its_force_quiescent(struct gicv3_its_data *data)
|
||||
{
|
||||
unsigned int count = WAIT_QUIESCENT;
|
||||
uint32_t reg = sys_read32(data->base + GITS_CTLR);
|
||||
|
||||
if (GITS_CTLR_ENABLED_GET(reg)) {
|
||||
/* Disable ITS */
|
||||
reg &= ~MASK(GITS_CTLR_ENABLED);
|
||||
sys_write32(reg, data->base + GITS_CTLR);
|
||||
}
|
||||
|
||||
while (1) {
|
||||
if (GITS_CTLR_QUIESCENT_GET(reg)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
count--;
|
||||
if (!count) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
k_msleep(1);
|
||||
reg = sys_read32(data->base + GITS_CTLR);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const char *const its_base_type_string[] = {
|
||||
[GITS_BASER_TYPE_DEVICE] = "Devices",
|
||||
[GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections",
|
||||
};
|
||||
|
||||
static int its_alloc_tables(struct gicv3_its_data *data)
|
||||
{
|
||||
unsigned int device_ids = GITS_TYPER_DEVBITS_GET(sys_read64(data->base + GITS_TYPER)) + 1;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < GITS_BASER_NR_REGS; ++i) {
|
||||
uint64_t reg = sys_read64(data->base + GITS_BASER(i));
|
||||
unsigned int type = GITS_BASER_TYPE_GET(reg);
|
||||
size_t page_size, entry_size, page_cnt;
|
||||
void *alloc_addr;
|
||||
|
||||
entry_size = GITS_BASER_ENTRY_SIZE_GET(reg) + 1;
|
||||
|
||||
switch (GITS_BASER_PAGE_SIZE_GET(reg)) {
|
||||
case GITS_BASER_PAGE_SIZE_4K:
|
||||
page_size = SIZE_4K;
|
||||
break;
|
||||
case GITS_BASER_PAGE_SIZE_16K:
|
||||
page_size = SIZE_16K;
|
||||
break;
|
||||
case GITS_BASER_PAGE_SIZE_64K:
|
||||
page_size = SIZE_64K;
|
||||
break;
|
||||
default:
|
||||
page_size = SIZE_4K;
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case GITS_BASER_TYPE_DEVICE:
|
||||
/*
|
||||
* TOFIX: implement & force support for indirect page table
|
||||
* 32bit device_ids can lead to 32GiB allocation
|
||||
* with PCIe, it should be 16bits max
|
||||
*/
|
||||
page_cnt = ROUND_UP(entry_size << device_ids, page_size) / page_size;
|
||||
break;
|
||||
case GITS_BASER_TYPE_COLLECTION:
|
||||
page_cnt = ROUND_UP(entry_size * CONFIG_MP_NUM_CPUS, page_size) / page_size;
|
||||
break;
|
||||
default:
|
||||
continue;
|
||||
}
|
||||
|
||||
LOG_INF("Allocating %s table of %ldx%ldK pages (%ld bytes entry)",
|
||||
its_base_type_string[type], page_cnt, page_size / 1024, entry_size);
|
||||
|
||||
alloc_addr = k_aligned_alloc(page_size, page_size * page_cnt);
|
||||
if (!alloc_addr) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
memset(alloc_addr, 0, page_size * page_cnt);
|
||||
|
||||
switch (page_size) {
|
||||
case SIZE_4K:
|
||||
reg = MASK_SET(GITS_BASER_PAGE_SIZE_4K, GITS_BASER_PAGE_SIZE);
|
||||
break;
|
||||
case SIZE_16K:
|
||||
reg = MASK_SET(GITS_BASER_PAGE_SIZE_16K, GITS_BASER_PAGE_SIZE);
|
||||
break;
|
||||
case SIZE_64K:
|
||||
reg = MASK_SET(GITS_BASER_PAGE_SIZE_64K, GITS_BASER_PAGE_SIZE);
|
||||
break;
|
||||
}
|
||||
|
||||
reg |= MASK_SET(page_cnt - 1, GITS_BASER_SIZE);
|
||||
reg |= MASK_SET(GIC_BASER_SHARE_INNER, GITS_BASER_SHAREABILITY);
|
||||
reg |= MASK_SET((uintptr_t)alloc_addr >> GITS_BASER_ADDR_SHIFT, GITS_BASER_ADDR);
|
||||
reg |= MASK_SET(GIC_BASER_CACHE_INNERLIKE, GITS_BASER_OUTER_CACHE);
|
||||
reg |= MASK_SET(GIC_BASER_CACHE_RAWAWB, GITS_BASER_INNER_CACHE);
|
||||
reg |= MASK_SET(0, GITS_BASER_INDIRECT);
|
||||
reg |= MASK_SET(1, GITS_BASER_VALID);
|
||||
|
||||
sys_write64(reg, data->base + GITS_BASER(i));
|
||||
|
||||
/* TOFIX: check page size & SHAREABILITY validity after write */
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool its_queue_full(struct gicv3_its_data *data)
|
||||
{
|
||||
int widx;
|
||||
int ridx;
|
||||
|
||||
widx = data->cmd_write - data->cmd_base;
|
||||
ridx = sys_read32(data->base + GITS_CREADR) / sizeof(struct its_cmd_block);
|
||||
|
||||
/* This is incredibly unlikely to happen, unless the ITS locks up. */
|
||||
return (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx);
|
||||
}
|
||||
|
||||
static struct its_cmd_block *its_allocate_entry(struct gicv3_its_data *data)
|
||||
{
|
||||
struct its_cmd_block *cmd;
|
||||
unsigned int count = 1000000; /* 1s! */
|
||||
|
||||
while (its_queue_full(data)) {
|
||||
count--;
|
||||
if (!count) {
|
||||
LOG_ERR("ITS queue not draining");
|
||||
return NULL;
|
||||
}
|
||||
k_usleep(1);
|
||||
}
|
||||
|
||||
cmd = data->cmd_write++;
|
||||
|
||||
/* Handle queue wrapping */
|
||||
if (data->cmd_write == (data->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) {
|
||||
data->cmd_write = data->cmd_base;
|
||||
}
|
||||
|
||||
/* Clear command */
|
||||
cmd->raw_cmd[0] = 0;
|
||||
cmd->raw_cmd[1] = 0;
|
||||
cmd->raw_cmd[2] = 0;
|
||||
cmd->raw_cmd[3] = 0;
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
static int its_post_command(struct gicv3_its_data *data, struct its_cmd_block *cmd)
|
||||
{
|
||||
uint64_t wr_idx, rd_idx, idx;
|
||||
unsigned int count = 1000000; /* 1s! */
|
||||
|
||||
wr_idx = (data->cmd_write - data->cmd_base) * sizeof(*cmd);
|
||||
rd_idx = sys_read32(data->base + GITS_CREADR);
|
||||
|
||||
dsb();
|
||||
|
||||
sys_write32(wr_idx, data->base + GITS_CWRITER);
|
||||
|
||||
while (1) {
|
||||
idx = sys_read32(data->base + GITS_CREADR);
|
||||
|
||||
if (idx == wr_idx) {
|
||||
break;
|
||||
}
|
||||
|
||||
count--;
|
||||
if (!count) {
|
||||
LOG_ERR("ITS queue timeout (rd %lld => %lld => wr %lld)",
|
||||
rd_idx, idx, wr_idx);
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
k_usleep(1);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int its_send_sync_cmd(struct gicv3_its_data *data, uintptr_t rd_addr)
|
||||
{
|
||||
struct its_cmd_block *cmd = its_allocate_entry(data);
|
||||
|
||||
if (!cmd) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
cmd->raw_cmd[0] = MASK_SET(GITS_CMD_ID_SYNC, GITS_CMD_ID);
|
||||
cmd->raw_cmd[2] = MASK_SET(rd_addr >> GITS_CMD_RDBASE_ALIGN, GITS_CMD_RDBASE);
|
||||
|
||||
return its_post_command(data, cmd);
|
||||
}
|
||||
|
||||
static int its_send_mapc_cmd(struct gicv3_its_data *data, uint32_t icid,
|
||||
uintptr_t rd_addr, bool valid)
|
||||
{
|
||||
struct its_cmd_block *cmd = its_allocate_entry(data);
|
||||
|
||||
if (!cmd) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
cmd->raw_cmd[0] = MASK_SET(GITS_CMD_ID_MAPC, GITS_CMD_ID);
|
||||
cmd->raw_cmd[2] = MASK_SET(icid, GITS_CMD_ICID) |
|
||||
MASK_SET(rd_addr >> GITS_CMD_RDBASE_ALIGN, GITS_CMD_RDBASE) |
|
||||
MASK_SET(valid ? 1 : 0, GITS_CMD_VALID);
|
||||
|
||||
return its_post_command(data, cmd);
|
||||
}
|
||||
|
||||
static int its_send_mapd_cmd(struct gicv3_its_data *data, uint32_t device_id,
|
||||
uint32_t size, uintptr_t itt_addr, bool valid)
|
||||
{
|
||||
struct its_cmd_block *cmd = its_allocate_entry(data);
|
||||
|
||||
if (!cmd) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
cmd->raw_cmd[0] = MASK_SET(GITS_CMD_ID_MAPD, GITS_CMD_ID) |
|
||||
MASK_SET(device_id, GITS_CMD_DEVICEID);
|
||||
cmd->raw_cmd[1] = MASK_SET(size, GITS_CMD_SIZE);
|
||||
cmd->raw_cmd[2] = MASK_SET(itt_addr >> GITS_CMD_ITTADDR_ALIGN, GITS_CMD_ITTADDR) |
|
||||
MASK_SET(valid ? 1 : 0, GITS_CMD_VALID);
|
||||
|
||||
return its_post_command(data, cmd);
|
||||
}
|
||||
|
||||
static int its_send_mapti_cmd(struct gicv3_its_data *data, uint32_t device_id,
|
||||
uint32_t event_id, uint32_t intid, uint32_t icid)
|
||||
{
|
||||
struct its_cmd_block *cmd = its_allocate_entry(data);
|
||||
|
||||
if (!cmd) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
cmd->raw_cmd[0] = MASK_SET(GITS_CMD_ID_MAPTI, GITS_CMD_ID) |
|
||||
MASK_SET(device_id, GITS_CMD_DEVICEID);
|
||||
cmd->raw_cmd[1] = MASK_SET(event_id, GITS_CMD_EVENTID) |
|
||||
MASK_SET(intid, GITS_CMD_PINTID);
|
||||
cmd->raw_cmd[2] = MASK_SET(icid, GITS_CMD_ICID);
|
||||
|
||||
return its_post_command(data, cmd);
|
||||
}
|
||||
|
||||
static int its_send_int_cmd(struct gicv3_its_data *data, uint32_t device_id,
|
||||
uint32_t event_id)
|
||||
{
|
||||
struct its_cmd_block *cmd = its_allocate_entry(data);
|
||||
|
||||
if (!cmd) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
cmd->raw_cmd[0] = MASK_SET(GITS_CMD_ID_INT, GITS_CMD_ID) |
|
||||
MASK_SET(device_id, GITS_CMD_DEVICEID);
|
||||
cmd->raw_cmd[1] = MASK_SET(event_id, GITS_CMD_EVENTID);
|
||||
|
||||
return its_post_command(data, cmd);
|
||||
}
|
||||
|
||||
static int its_send_invall_cmd(struct gicv3_its_data *data, uint32_t icid)
|
||||
{
|
||||
struct its_cmd_block *cmd = its_allocate_entry(data);
|
||||
|
||||
if (!cmd) {
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
cmd->raw_cmd[0] = MASK_SET(GITS_CMD_ID_INVALL, GITS_CMD_ID);
|
||||
cmd->raw_cmd[2] = MASK_SET(icid, GITS_CMD_ICID);
|
||||
|
||||
return its_post_command(data, cmd);
|
||||
}
|
||||
|
||||
static int gicv3_its_send_int(const struct device *dev, uint32_t device_id, uint32_t event_id)
|
||||
{
|
||||
struct gicv3_its_data *data = (struct gicv3_its_data *)dev->data;
|
||||
|
||||
/* TOFIX check device_id & event_id bounds */
|
||||
|
||||
return its_send_int_cmd(data, device_id, event_id);
|
||||
}
|
||||
|
||||
static void its_setup_cmd_queue(const struct device *dev)
|
||||
{
|
||||
const struct gicv3_its_config *cfg = (const struct gicv3_its_config *)dev->config;
|
||||
struct gicv3_its_data *data = (struct gicv3_its_data *)dev->data;
|
||||
uint64_t reg = 0;
|
||||
|
||||
/* Zero out cmd table */
|
||||
memset(cfg->cmd_queue, 0, cfg->cmd_queue_size);
|
||||
|
||||
reg |= MASK_SET(cfg->cmd_queue_size / SIZE_4K, GITS_CBASER_SIZE);
|
||||
reg |= MASK_SET(GIC_BASER_SHARE_INNER, GITS_CBASER_SHAREABILITY);
|
||||
reg |= MASK_SET((uintptr_t)cfg->cmd_queue >> GITS_CBASER_ADDR_SHIFT, GITS_CBASER_ADDR);
|
||||
reg |= MASK_SET(GIC_BASER_CACHE_RAWAWB, GITS_CBASER_OUTER_CACHE);
|
||||
reg |= MASK_SET(GIC_BASER_CACHE_RAWAWB, GITS_CBASER_INNER_CACHE);
|
||||
reg |= MASK_SET(1, GITS_CBASER_VALID);
|
||||
|
||||
sys_write64(reg, data->base + GITS_CBASER);
|
||||
|
||||
data->cmd_base = (struct its_cmd_block *)cfg->cmd_queue;
|
||||
data->cmd_write = data->cmd_base;
|
||||
|
||||
LOG_INF("Allocated %ld entries for command table", ITS_CMD_QUEUE_NR_ENTRIES);
|
||||
|
||||
sys_write64(0, data->base + GITS_CWRITER);
|
||||
}
|
||||
|
||||
static uintptr_t gicv3_rdist_get_rdbase(const struct device *dev, unsigned int cpuid)
|
||||
{
|
||||
struct gicv3_its_data *data = (struct gicv3_its_data *)dev->data;
|
||||
uint64_t typer = sys_read64(data->base + GITS_TYPER);
|
||||
|
||||
if (GITS_TYPER_PTA_GET(typer)) {
|
||||
return gic_rdists[cpuid];
|
||||
} else {
|
||||
return GICR_TYPER_PROCESSOR_NUMBER_GET(sys_read64(gic_rdists[cpuid] + GICR_TYPER));
|
||||
}
|
||||
}
|
||||
|
||||
static int gicv3_its_map_intid(const struct device *dev, uint32_t device_id, uint32_t event_id,
|
||||
unsigned int intid)
|
||||
{
|
||||
struct gicv3_its_data *data = (struct gicv3_its_data *)dev->data;
|
||||
int ret;
|
||||
|
||||
/* TOFIX check device_id, event_id & intid bounds */
|
||||
|
||||
if (intid < 8192) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* The CPU id directly maps as ICID for the currect CPU redistributor */
|
||||
ret = its_send_mapti_cmd(data, device_id, event_id, intid, arch_curr_cpu()->id);
|
||||
if (ret) {
|
||||
LOG_ERR("Failed to map eventid %d to intid %d for deviceid %x",
|
||||
event_id, intid, device_id);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return its_send_sync_cmd(data, gicv3_rdist_get_rdbase(dev, arch_curr_cpu()->id));
|
||||
}
|
||||
|
||||
static inline int fls_z(unsigned int x)
|
||||
{
|
||||
unsigned int bits = sizeof(x) * 8;
|
||||
unsigned int cmp = 1 << (bits - 1);
|
||||
|
||||
while (bits) {
|
||||
if (x & cmp) {
|
||||
return bits;
|
||||
}
|
||||
cmp >>= 1;
|
||||
bits--;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int gicv3_its_init_device_id(const struct device *dev, uint32_t device_id,
|
||||
unsigned int nites)
|
||||
{
|
||||
struct gicv3_its_data *data = (struct gicv3_its_data *)dev->data;
|
||||
size_t entry_size, alloc_size;
|
||||
int nr_ites;
|
||||
void *itt;
|
||||
int ret;
|
||||
|
||||
/* TOFIX check device_id & nites bounds */
|
||||
|
||||
entry_size = GITS_TYPER_ITT_ENTRY_SIZE_GET(sys_read64(data->base + GITS_TYPER)) + 1;
|
||||
|
||||
/* ITT must be of power of 2 */
|
||||
nr_ites = MAX(2, nites);
|
||||
alloc_size = ROUND_UP(nr_ites * entry_size, 256);
|
||||
|
||||
LOG_INF("Allocating ITT for DeviceID %x and %d vectors (%ld bytes entry)",
|
||||
device_id, nr_ites, entry_size);
|
||||
|
||||
itt = k_aligned_alloc(256, alloc_size);
|
||||
if (!itt) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/* size is log2(ites) - 1, equivalent to (fls(ites) - 1) - 1 */
|
||||
ret = its_send_mapd_cmd(data, device_id, fls_z(nr_ites) - 2, (uintptr_t)itt, true);
|
||||
if (ret) {
|
||||
LOG_ERR("Failed to map device id %x ITT table", device_id);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static unsigned int gicv3_its_alloc_intid(const struct device *dev)
|
||||
{
|
||||
return atomic_inc(&nlpi_intid);
|
||||
}
|
||||
|
||||
#define ITS_RDIST_MAP(n) \
|
||||
{ \
|
||||
const struct device *dev = DEVICE_DT_INST_GET(n); \
|
||||
struct gicv3_its_data *data; \
|
||||
int ret; \
|
||||
\
|
||||
if (dev) { \
|
||||
data = (struct gicv3_its_data *) dev->data; \
|
||||
ret = its_send_mapc_cmd(data, arch_curr_cpu()->id, \
|
||||
gicv3_rdist_get_rdbase(dev, arch_curr_cpu()->id), \
|
||||
true); \
|
||||
if (ret) { \
|
||||
LOG_ERR("Failed to map CPU%d redistributor", \
|
||||
arch_curr_cpu()->id); \
|
||||
} \
|
||||
} \
|
||||
}
|
||||
|
||||
void its_rdist_map(void)
|
||||
{
|
||||
DT_INST_FOREACH_STATUS_OKAY(ITS_RDIST_MAP)
|
||||
}
|
||||
|
||||
#define ITS_RDIST_INVALL(n) \
|
||||
{ \
|
||||
const struct device *dev = DEVICE_DT_INST_GET(n); \
|
||||
struct gicv3_its_data *data; \
|
||||
int ret; \
|
||||
\
|
||||
if (dev) { \
|
||||
data = (struct gicv3_its_data *) dev->data; \
|
||||
ret = its_send_invall_cmd(data, arch_curr_cpu()->id); \
|
||||
if (ret) { \
|
||||
LOG_ERR("Failed to sync RDIST LPI cache for CPU%d", \
|
||||
arch_curr_cpu()->id); \
|
||||
} \
|
||||
\
|
||||
its_send_sync_cmd(data, \
|
||||
gicv3_rdist_get_rdbase(dev, arch_curr_cpu()->id)); \
|
||||
} \
|
||||
}
|
||||
|
||||
void its_rdist_invall(void)
|
||||
{
|
||||
DT_INST_FOREACH_STATUS_OKAY(ITS_RDIST_INVALL)
|
||||
}
|
||||
|
||||
static int gicv3_its_init(const struct device *dev)
|
||||
{
|
||||
const struct gicv3_its_config *cfg = (const struct gicv3_its_config *)dev->config;
|
||||
struct gicv3_its_data *data = (struct gicv3_its_data *)dev->data;
|
||||
uint32_t reg;
|
||||
int ret;
|
||||
|
||||
device_map(&data->base, cfg->base_addr, cfg->base_size, K_MEM_CACHE_NONE);
|
||||
|
||||
ret = its_force_quiescent(data);
|
||||
if (ret) {
|
||||
LOG_ERR("Failed to quiesce, giving up");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = its_alloc_tables(data);
|
||||
if (ret) {
|
||||
LOG_ERR("Failed to allocate tables, giving up");
|
||||
return ret;
|
||||
}
|
||||
|
||||
its_setup_cmd_queue(dev);
|
||||
|
||||
reg = sys_read32(data->base + GITS_CTLR);
|
||||
reg |= MASK_SET(1, GITS_CTLR_ENABLED);
|
||||
sys_write32(reg, data->base + GITS_CTLR);
|
||||
|
||||
/* Map the boot CPU id to the CPU redistributor */
|
||||
ret = its_send_mapc_cmd(data, arch_curr_cpu()->id,
|
||||
gicv3_rdist_get_rdbase(dev, arch_curr_cpu()->id), true);
|
||||
if (ret) {
|
||||
LOG_ERR("Failed to map boot CPU redistributor");
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct its_driver_api gicv3_its_api = {
|
||||
.alloc_intid = gicv3_its_alloc_intid,
|
||||
.setup_deviceid = gicv3_its_init_device_id,
|
||||
.map_intid = gicv3_its_map_intid,
|
||||
.send_int = gicv3_its_send_int,
|
||||
};
|
||||
|
||||
#define GICV3_ITS_INIT(n) \
|
||||
static struct its_cmd_block gicv3_its_cmd##n[ITS_CMD_QUEUE_NR_ENTRIES] \
|
||||
__aligned(ITS_CMD_QUEUE_SIZE); \
|
||||
static struct gicv3_its_data gicv3_its_data##n; \
|
||||
static const struct gicv3_its_config gicv3_its_config##n = { \
|
||||
.base_addr = DT_INST_REG_ADDR(n), \
|
||||
.base_size = DT_INST_REG_SIZE(n), \
|
||||
.cmd_queue = gicv3_its_cmd##n, \
|
||||
.cmd_queue_size = sizeof(gicv3_its_cmd##n), \
|
||||
}; \
|
||||
DEVICE_DT_INST_DEFINE(n, &gicv3_its_init, NULL, \
|
||||
&gicv3_its_data##n, \
|
||||
&gicv3_its_config##n, \
|
||||
POST_KERNEL, \
|
||||
CONFIG_KERNEL_INIT_PRIORITY_DEFAULT, \
|
||||
&gicv3_its_api);
|
||||
|
||||
DT_INST_FOREACH_STATUS_OKAY(GICV3_ITS_INIT)
|
|
@ -9,17 +9,18 @@
|
|||
|
||||
#include <zephyr/types.h>
|
||||
#include <device.h>
|
||||
#include <sys/atomic.h>
|
||||
|
||||
/* Cache and Share ability for ITS & Redistributor LPI state tables */
|
||||
#define GIC_BASER_CACHE_NGNRNE 0x0UL /* Device-nGnRnE */
|
||||
#define GIC_BASER_CACHE_INNERLIKE 0x0UL /* Same as Inner Cacheability. */
|
||||
#define GIC_BASER_CACHE_NCACHEABLE 0x1UL /* Normal Outer Non-cacheable */
|
||||
#define GIC_BASER_CACHE_RAWT 0x2UL /* Normal Outer Cacheable Read-allocate, Write-through */
|
||||
#define GIC_BASER_CACHE_RAWB 0x3UL /* Normal Outer Cacheable Read-allocate, Write-back */
|
||||
#define GIC_BASER_CACHE_WAWT 0x4UL /* Normal Outer Cacheable Write-allocate, Write-through */
|
||||
#define GIC_BASER_CACHE_WAWB 0x5UL /* Normal Outer Cacheable Write-allocate, Write-back */
|
||||
#define GIC_BASER_CACHE_RAWAWT 0x6UL /* Normal Outer Cacheable Read-allocate, Write-allocate, Write-through */
|
||||
#define GIC_BASER_CACHE_RAWAWB 0x7UL /* Normal Outer Cacheable Read-allocate, Write-allocate, Write-back */
|
||||
#define GIC_BASER_CACHE_NCACHEABLE 0x1UL /* Non-cacheable */
|
||||
#define GIC_BASER_CACHE_RAWT 0x2UL /* Cacheable R-allocate, W-through */
|
||||
#define GIC_BASER_CACHE_RAWB 0x3UL /* Cacheable R-allocate, W-back */
|
||||
#define GIC_BASER_CACHE_WAWT 0x4UL /* Cacheable W-allocate, W-through */
|
||||
#define GIC_BASER_CACHE_WAWB 0x5UL /* Cacheable W-allocate, W-back */
|
||||
#define GIC_BASER_CACHE_RAWAWT 0x6UL /* Cacheable R-allocate, W-allocate, W-through */
|
||||
#define GIC_BASER_CACHE_RAWAWB 0x7UL /* Cacheable R-allocate, W-allocate, W-back */
|
||||
#define GIC_BASER_SHARE_NO 0x0UL /* Non-shareable */
|
||||
#define GIC_BASER_SHARE_INNER 0x1UL /* Inner Shareable */
|
||||
#define GIC_BASER_SHARE_OUTER 0x2UL /* Outer Shareable */
|
||||
|
@ -60,6 +61,11 @@
|
|||
#define GICR_CTLR_ENABLE_LPIS BIT(0)
|
||||
#define GICR_CTLR_RWP 3
|
||||
|
||||
/* GICR_TYPER */
|
||||
#define GICR_TYPER_PROCESSOR_NUMBER_SHIFT 8
|
||||
#define GICR_TYPER_PROCESSOR_NUMBER_MASK 0xFFFFUL
|
||||
#define GICR_TYPER_PROCESSOR_NUMBER_GET(_val) MASK_GET(_val, GICR_TYPER_PROCESSOR_NUMBER)
|
||||
|
||||
/* GICR_WAKER */
|
||||
#define GICR_WAKER_PS 1
|
||||
#define GICR_WAKER_CA 2
|
||||
|
@ -90,4 +96,174 @@
|
|||
#define GIC_DIST_IROUTER 0x6000
|
||||
#define IROUTER(base, n) (base + GIC_DIST_IROUTER + (n) * 8)
|
||||
|
||||
/*
|
||||
* ITS registers, offsets from ITS_base
|
||||
*/
|
||||
#define GITS_CTLR 0x0000
|
||||
#define GITS_IIDR 0x0004
|
||||
#define GITS_TYPER 0x0008
|
||||
#define GITS_STATUSR 0x0040
|
||||
#define GITS_UMSIR 0x0048
|
||||
#define GITS_CBASER 0x0080
|
||||
#define GITS_CWRITER 0x0088
|
||||
#define GITS_CREADR 0x0090
|
||||
#define GITS_BASER(n) (0x0100 + ((n) * 8))
|
||||
|
||||
#define GITS_TRANSLATER 0x10040
|
||||
|
||||
/* ITS CTLR register */
|
||||
#define GITS_CTLR_ENABLED_SHIFT 0
|
||||
#define GITS_CTLR_ENABLED_MASK 0x1UL
|
||||
#define GITS_CTLR_ITS_NUMBER_SHIFT 4
|
||||
#define GITS_CTLR_ITS_NUMBER_MASK 0xfUL
|
||||
#define GITS_CTLR_QUIESCENT_SHIFT 31
|
||||
#define GITS_CTLR_QUIESCENT_MASK 0x1UL
|
||||
|
||||
#define GITS_CTLR_ENABLED_GET(_val) MASK_GET(_val, GITS_CTLR_ENABLED)
|
||||
#define GITS_CTLR_QUIESCENT_GET(_val) MASK_GET(_val, GITS_CTLR_QUIESCENT)
|
||||
|
||||
/* ITS TYPER register */
|
||||
#define GITS_TYPER_PHY_SHIFT 0
|
||||
#define GITS_TYPER_PHY_MASK 0x1UL
|
||||
#define GITS_TYPER_VIRT_SHIFT 1
|
||||
#define GITS_TYPER_VIRT_MASK 0x1UL
|
||||
#define GITS_TYPER_ITT_ENTRY_SIZE_SHIFT 4
|
||||
#define GITS_TYPER_ITT_ENTRY_SIZE_MASK 0xfUL
|
||||
#define GITS_TYPER_IDBITS_SHIFT 8
|
||||
#define GITS_TYPER_IDBITS_MASK 0x1fUL
|
||||
#define GITS_TYPER_DEVBITS_SHIFT 13
|
||||
#define GITS_TYPER_DEVBITS_MASK 0x1fUL
|
||||
#define GITS_TYPER_SEIS_SHIFT 18
|
||||
#define GITS_TYPER_SEIS_MASK 0x1UL
|
||||
#define GITS_TYPER_PTA_SHIFT 19
|
||||
#define GITS_TYPER_PTA_MASK 0x1UL
|
||||
#define GITS_TYPER_HCC_SHIFT 24
|
||||
#define GITS_TYPER_HCC_MASK 0xffUL
|
||||
#define GITS_TYPER_CIDBITS_SHIFT 32
|
||||
#define GITS_TYPER_CIDBITS_MASK 0xfUL
|
||||
#define GITS_TYPER_CIL_SHIFT 36
|
||||
#define GITS_TYPER_CIL_MASK 0x1UL
|
||||
|
||||
#define GITS_TYPER_ITT_ENTRY_SIZE_GET(_val) MASK_GET(_val, GITS_TYPER_ITT_ENTRY_SIZE)
|
||||
#define GITS_TYPER_PTA_GET(_val) MASK_GET(_val, GITS_TYPER_PTA)
|
||||
#define GITS_TYPER_HCC_GET(_val) MASK_GET(_val, GITS_TYPER_HCC)
|
||||
#define GITS_TYPER_DEVBITS_GET(_val) MASK_GET(_val, GITS_TYPER_DEVBITS)
|
||||
|
||||
/* ITS COMMON BASER / CBASER register */
|
||||
|
||||
/* ITS CBASER register */
|
||||
#define GITS_CBASER_SIZE_SHIFT 0
|
||||
#define GITS_CBASER_SIZE_MASK 0xffUL
|
||||
#define GITS_CBASER_SHAREABILITY_SHIFT 10
|
||||
#define GITS_CBASER_SHAREABILITY_MASK 0x3UL
|
||||
#define GITS_CBASER_ADDR_SHIFT 12
|
||||
#define GITS_CBASER_ADDR_MASK 0xfffffffffUL
|
||||
#define GITS_CBASER_OUTER_CACHE_SHIFT 53
|
||||
#define GITS_CBASER_OUTER_CACHE_MASK 0x7UL
|
||||
#define GITS_CBASER_INNER_CACHE_SHIFT 59
|
||||
#define GITS_CBASER_INNER_CACHE_MASK 0x7UL
|
||||
#define GITS_CBASER_VALID_SHIFT 63
|
||||
#define GITS_CBASER_VALID_MASK 0x1UL
|
||||
|
||||
/* ITS BASER<n> register */
|
||||
#define GITS_BASER_SIZE_SHIFT 0
|
||||
#define GITS_BASER_SIZE_MASK 0xffUL
|
||||
#define GITS_BASER_PAGE_SIZE_SHIFT 8
|
||||
#define GITS_BASER_PAGE_SIZE_MASK 0x3UL
|
||||
#define GITS_BASER_PAGE_SIZE_4K 0
|
||||
#define GITS_BASER_PAGE_SIZE_16K 1
|
||||
#define GITS_BASER_PAGE_SIZE_64K 2
|
||||
#define GITS_BASER_SHAREABILITY_SHIFT 10
|
||||
#define GITS_BASER_SHAREABILITY_MASK 0x3UL
|
||||
#define GITS_BASER_ADDR_SHIFT 12
|
||||
#define GITS_BASER_ADDR_MASK 0xfffffffff
|
||||
#define GITS_BASER_ENTRY_SIZE_SHIFT 48
|
||||
#define GITS_BASER_ENTRY_SIZE_MASK 0x1fUL
|
||||
#define GITS_BASER_OUTER_CACHE_SHIFT 53
|
||||
#define GITS_BASER_OUTER_CACHE_MASK 0x7UL
|
||||
#define GITS_BASER_TYPE_SHIFT 56
|
||||
#define GITS_BASER_TYPE_MASK 0x7UL
|
||||
#define GITS_BASER_INNER_CACHE_SHIFT 59
|
||||
#define GITS_BASER_INNER_CACHE_MASK 0x7UL
|
||||
#define GITS_BASER_INDIRECT_SHIFT 62
|
||||
#define GITS_BASER_INDIRECT_MASK 0x1UL
|
||||
#define GITS_BASER_VALID_SHIFT 63
|
||||
#define GITS_BASER_VALID_MASK 0x1UL
|
||||
|
||||
#define GITS_BASER_TYPE_NONE 0
|
||||
#define GITS_BASER_TYPE_DEVICE 1
|
||||
#define GITS_BASER_TYPE_COLLECTION 4
|
||||
|
||||
#define GITS_BASER_TYPE_GET(_val) MASK_GET(_val, GITS_BASER_TYPE)
|
||||
#define GITS_BASER_PAGE_SIZE_GET(_val) MASK_GET(_val, GITS_BASER_PAGE_SIZE)
|
||||
#define GITS_BASER_ENTRY_SIZE_GET(_val) MASK_GET(_val, GITS_BASER_ENTRY_SIZE)
|
||||
#define GITS_BASER_INDIRECT_GET(_val) MASK_GET(_val, GITS_BASER_INDIRECT)
|
||||
|
||||
#define GITS_BASER_NR_REGS 8
|
||||
|
||||
/* ITS Commands */
|
||||
|
||||
#define GITS_CMD_ID_MOVI 0x01
|
||||
#define GITS_CMD_ID_INT 0x03
|
||||
#define GITS_CMD_ID_CLEAR 0x04
|
||||
#define GITS_CMD_ID_SYNC 0x05
|
||||
#define GITS_CMD_ID_MAPD 0x08
|
||||
#define GITS_CMD_ID_MAPC 0x09
|
||||
#define GITS_CMD_ID_MAPTI 0x0a
|
||||
#define GITS_CMD_ID_MAPI 0x0b
|
||||
#define GITS_CMD_ID_INV 0x0c
|
||||
#define GITS_CMD_ID_INVALL 0x0d
|
||||
#define GITS_CMD_ID_MOVALL 0x0e
|
||||
#define GITS_CMD_ID_DISCARD 0x0f
|
||||
|
||||
#define GITS_CMD_ID_OFFSET 0
|
||||
#define GITS_CMD_ID_SHIFT 0
|
||||
#define GITS_CMD_ID_MASK 0xffUL
|
||||
|
||||
#define GITS_CMD_DEVICEID_OFFSET 0
|
||||
#define GITS_CMD_DEVICEID_SHIFT 32
|
||||
#define GITS_CMD_DEVICEID_MASK 0xffffffffUL
|
||||
|
||||
#define GITS_CMD_SIZE_OFFSET 1
|
||||
#define GITS_CMD_SIZE_SHIFT 0
|
||||
#define GITS_CMD_SIZE_MASK 0x1fUL
|
||||
|
||||
#define GITS_CMD_EVENTID_OFFSET 1
|
||||
#define GITS_CMD_EVENTID_SHIFT 0
|
||||
#define GITS_CMD_EVENTID_MASK 0xffffffffUL
|
||||
|
||||
#define GITS_CMD_PINTID_OFFSET 1
|
||||
#define GITS_CMD_PINTID_SHIFT 32
|
||||
#define GITS_CMD_PINTID_MASK 0xffffffffUL
|
||||
|
||||
#define GITS_CMD_ICID_OFFSET 2
|
||||
#define GITS_CMD_ICID_SHIFT 0
|
||||
#define GITS_CMD_ICID_MASK 0xffffUL
|
||||
|
||||
#define GITS_CMD_ITTADDR_OFFSET 2
|
||||
#define GITS_CMD_ITTADDR_SHIFT 8
|
||||
#define GITS_CMD_ITTADDR_MASK 0xffffffffffUL
|
||||
#define GITS_CMD_ITTADDR_ALIGN GITS_CMD_ITTADDR_SHIFT
|
||||
#define GITS_CMD_ITTADDR_ALIGN_SZ (BIT(0) << GITS_CMD_ITTADDR_ALIGN)
|
||||
|
||||
#define GITS_CMD_RDBASE_OFFSET 2
|
||||
#define GITS_CMD_RDBASE_SHIFT 16
|
||||
#define GITS_CMD_RDBASE_MASK 0xffffffffUL
|
||||
#define GITS_CMD_RDBASE_ALIGN GITS_CMD_RDBASE_SHIFT
|
||||
|
||||
#define GITS_CMD_VALID_OFFSET 2
|
||||
#define GITS_CMD_VALID_SHIFT 63
|
||||
#define GITS_CMD_VALID_MASK 0x1UL
|
||||
|
||||
#define MASK(__basename) (__basename##_MASK << __basename##_SHIFT)
|
||||
#define MASK_SET(__val, __basename) (((__val) & __basename##_MASK) << __basename##_SHIFT)
|
||||
#define MASK_GET(__reg, __basename) (((__reg) >> __basename##_SHIFT) & __basename##_MASK)
|
||||
|
||||
#ifdef CONFIG_GIC_V3_ITS
|
||||
void its_rdist_map(void);
|
||||
void its_rdist_invall(void);
|
||||
|
||||
extern atomic_t nlpi_intid;
|
||||
#endif
|
||||
|
||||
#endif /* ZEPHYR_INCLUDE_DRIVERS_INTC_GICV3_PRIV_H_ */
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue