michaelh/zephyr
1
0
Fork 0
Code Releases Activity
zephyr/arch/arm/core/mmu/arm_mmu.c
Marek Vedral 93a4287115 arm: debug: Add GDB stub for aarch32 
This commit adds implementation of GDB stub for 32-bit ARM. It has been
tested only on the Zynq-7000 SoC and I would like to get any feedback
from others.

The stub still has these issues:

- To implement single stepping, it uses instruction address mismatch
  breakpoint, as recommended in ARMv7 reference. The breakpoint control
  register is configured (the state control fields) for the "PL0,
  Supervisor and System modes only" option. Otherwise the breakpoint
  would also halt the processor in abort mode, in which the stub loop
  runs. Zephyr kernel runs in the system mode. This works well until the
  kernel enables interrupts, as interrupt handlers typically run in
  Supervisor mode. Single stepping therefore sometimes "catches" a
  handler instead of the next application instruction. I have not tried
  User mode, because Cortex-A SoCs do not appear to have the
  ARCH_HAS_USERSPACE flag.

Cc: Michal Sojka <michal.sojka@cvut.cz>
Signed-off-by: Marek Vedral <marek.vedral@gmail.com>
2023-12-18 09:31:42 +01:00

1101 lines
40 KiB
C
Raw Permalink Blame History

/*
* ARMv7 MMU support
*
* This implementation supports the Short-descriptor translation
* table format. The standard page size is 4 kB, 1 MB sections
* are only used for mapping the code and data of the Zephyr image.
* Secure mode and PL1 is always assumed. LPAE and PXN extensions
* as well as TEX remapping are not supported. The AP[2:1] plus
* Access flag permissions model is used, as the AP[2:0] model is
* deprecated. As the AP[2:1] model can only disable write access,
* the read permission flag is always implied.
*
* Reference documentation:
* ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition,
* ARM document ID DDI0406C Rev. d, March 2018
*
* Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG
* SPDX-License-Identifier: Apache-2.0
*/
#include <zephyr/device.h>
#include <zephyr/init.h>
#include <zephyr/kernel.h>
#include <zephyr/linker/linker-defs.h>
#include <zephyr/logging/log.h>
#include <zephyr/sys/__assert.h>
#include <zephyr/sys/util.h>
#include <zephyr/kernel/mm.h>
#include <zephyr/sys/barrier.h>
#include <cmsis_core.h>
#include <zephyr/arch/arm/mmu/arm_mmu.h>
#include "arm_mmu_priv.h"
LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
/* Level 1 page table: always required, must be 16k-aligned */
static struct arm_mmu_l1_page_table
l1_page_table __aligned(KB(16)) = {0};
/*
* Array of level 2 page tables with 4k granularity:
* each table covers a range of 1 MB, the number of L2 tables
* is configurable.
*/
static struct arm_mmu_l2_page_table
l2_page_tables[CONFIG_ARM_MMU_NUM_L2_TABLES] __aligned(KB(1)) = {0};
/*
* For each level 2 page table, a separate dataset tracks
* if the respective table is in use, if so, to which 1 MB
* virtual address range it is assigned, and how many entries,
* each mapping a 4 kB page, it currently contains.
*/
static struct arm_mmu_l2_page_table_status
l2_page_tables_status[CONFIG_ARM_MMU_NUM_L2_TABLES] = {0};
/* Available L2 tables count & next free index for an L2 table request */
static uint32_t arm_mmu_l2_tables_free = CONFIG_ARM_MMU_NUM_L2_TABLES;
static uint32_t arm_mmu_l2_next_free_table;
/*
* Static definition of all code & data memory regions of the
* current Zephyr image. This information must be available &
* processed upon MMU initialization.
*/
static const struct arm_mmu_flat_range mmu_zephyr_ranges[] = {
/*
* Mark the zephyr execution regions (data, bss, noinit, etc.)
* cacheable, read / write and non-executable
*/
{ .name = "zephyr_data",
.start = (uint32_t)_image_ram_start,
.end = (uint32_t)_image_ram_end,
.attrs = MT_NORMAL | MATTR_SHARED |
MPERM_R | MPERM_W |
MATTR_CACHE_OUTER_WB_WA | MATTR_CACHE_INNER_WB_WA},
/* Mark text segment cacheable, read only and executable */
{ .name = "zephyr_code",
.start = (uint32_t)__text_region_start,
.end = (uint32_t)__text_region_end,
.attrs = MT_NORMAL | MATTR_SHARED |
/* The code needs to have write permission in order for
* software breakpoints (which modify instructions) to work
*/
#if defined(CONFIG_GDBSTUB)
MPERM_R | MPERM_X | MPERM_W |
#else
MPERM_R | MPERM_X |
#endif
MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA |
MATTR_MAY_MAP_L1_SECTION},
/* Mark rodata segment cacheable, read only and non-executable */
{ .name = "zephyr_rodata",
.start = (uint32_t)__rodata_region_start,
.end = (uint32_t)__rodata_region_end,
.attrs = MT_NORMAL | MATTR_SHARED |
MPERM_R |
MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA |
MATTR_MAY_MAP_L1_SECTION},
#ifdef CONFIG_NOCACHE_MEMORY
/* Mark nocache segment read / write and non-executable */
{ .name = "nocache",
.start = (uint32_t)_nocache_ram_start,
.end = (uint32_t)_nocache_ram_end,
.attrs = MT_STRONGLY_ORDERED |
MPERM_R | MPERM_W},
#endif
};
static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa,
struct arm_mmu_perms_attrs perms_attrs);
/**
* @brief Invalidates the TLB
* Helper function which invalidates the entire TLB. This action
* is performed whenever the MMU is (re-)enabled or changes to the
* page tables are made at run-time, as the TLB might contain entries
* which are no longer valid once the changes are applied.
*/
static void invalidate_tlb_all(void)
{
__set_TLBIALL(0); /* 0 = opc2 = invalidate entire TLB */
barrier_dsync_fence_full();
barrier_isync_fence_full();
}
/**
* @brief Returns a free level 2 page table
* Initializes and returns the next free L2 page table whenever
* a page is to be mapped in a 1 MB virtual address range that
* is not yet covered by a level 2 page table.
*
* @param va 32-bit virtual address to be mapped.
* @retval pointer to the L2 table now assigned to the 1 MB
* address range the target virtual address is in.
*/
static struct arm_mmu_l2_page_table *arm_mmu_assign_l2_table(uint32_t va)
{
struct arm_mmu_l2_page_table *l2_page_table;
__ASSERT(arm_mmu_l2_tables_free > 0,
"Cannot set up L2 page table for VA 0x%08X: "
"no more free L2 page tables available\n",
va);
__ASSERT(l2_page_tables_status[arm_mmu_l2_next_free_table].entries == 0,
"Cannot set up L2 page table for VA 0x%08X: "
"expected empty L2 table at index [%u], but the "
"entries value is %u\n",
va, arm_mmu_l2_next_free_table,
l2_page_tables_status[arm_mmu_l2_next_free_table].entries);
/*
* Store in the status dataset of the L2 table to be returned
* which 1 MB virtual address range it is being assigned to.
* Set the current page table entry count to 0.
*/
l2_page_tables_status[arm_mmu_l2_next_free_table].l1_index =
((va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK);
l2_page_tables_status[arm_mmu_l2_next_free_table].entries = 0;
l2_page_table = &l2_page_tables[arm_mmu_l2_next_free_table];
/*
* Decrement the available L2 page table count. As long as at
* least one more L2 table is available afterwards, update the
* L2 next free table index. If we're about to return the last
* available L2 table, calculating a next free table index is
* impossible.
*/
--arm_mmu_l2_tables_free;
if (arm_mmu_l2_tables_free > 0) {
do {
arm_mmu_l2_next_free_table = (arm_mmu_l2_next_free_table + 1) %
CONFIG_ARM_MMU_NUM_L2_TABLES;
} while (l2_page_tables_status[arm_mmu_l2_next_free_table].entries != 0);
}
return l2_page_table;
}
/**
* @brief Releases a level 2 page table
* Releases a level 2 page table, marking it as no longer in use.
* From that point on, it can be re-used for mappings in another
* 1 MB virtual address range. This function is called whenever
* it is determined during an unmap call at run-time that the page
* table entry count in the respective page table has reached 0.
*
* @param l2_page_table Pointer to L2 page table to be released.
*/
static void arm_mmu_release_l2_table(struct arm_mmu_l2_page_table *l2_page_table)
{
uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table);
l2_page_tables_status[l2_page_table_index].l1_index = 0;
if (arm_mmu_l2_tables_free == 0) {
arm_mmu_l2_next_free_table = l2_page_table_index;
}
++arm_mmu_l2_tables_free;
}
/**
* @brief Increments the page table entry counter of a L2 page table
* Increments the page table entry counter of a level 2 page table.
* Contains a check to ensure that no attempts are made to set up
* more page table entries than the table can hold.
*
* @param l2_page_table Pointer to the L2 page table whose entry
* counter shall be incremented.
*/
static void arm_mmu_inc_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table)
{
uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table);
__ASSERT(l2_page_tables_status[l2_page_table_index].entries < ARM_MMU_PT_L2_NUM_ENTRIES,
"Cannot increment entry count of the L2 page table at index "
"[%u] / addr %p / ref L1[%u]: maximum entry count already reached",
l2_page_table_index, l2_page_table,
l2_page_tables_status[l2_page_table_index].l1_index);
++l2_page_tables_status[l2_page_table_index].entries;
}
/**
* @brief Decrements the page table entry counter of a L2 page table
* Decrements the page table entry counter of a level 2 page table.
* Contains a check to ensure that no attempts are made to remove
* entries from the respective table that aren't actually there.
*
* @param l2_page_table Pointer to the L2 page table whose entry
* counter shall be decremented.
*/
static void arm_mmu_dec_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table)
{
uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table);
__ASSERT(l2_page_tables_status[l2_page_table_index].entries > 0,
"Cannot decrement entry count of the L2 page table at index "
"[%u] / addr %p / ref L1[%u]: entry count is already zero",
l2_page_table_index, l2_page_table,
l2_page_tables_status[l2_page_table_index].l1_index);
if (--l2_page_tables_status[l2_page_table_index].entries == 0) {
arm_mmu_release_l2_table(l2_page_table);
}
}
/**
* @brief Converts memory attributes and permissions to MMU format
* Converts memory attributes and permissions as used in the boot-
* time memory mapping configuration data array (MT_..., MATTR_...,
* MPERM_...) to the equivalent bit (field) values used in the MMU's
* L1 and L2 page table entries. Contains plausibility checks.
*
* @param attrs type/attribute/permissions flags word obtained from
* an entry of the mmu_config mapping data array.
* @retval A struct containing the information from the input flags
* word converted to the bits / bit fields used in L1 and
* L2 page table entries.
*/
static struct arm_mmu_perms_attrs arm_mmu_convert_attr_flags(uint32_t attrs)
{
struct arm_mmu_perms_attrs perms_attrs = {0};
__ASSERT(((attrs & MT_MASK) > 0),
"Cannot convert attrs word to PTE control bits: no "
"memory type specified");
__ASSERT(!((attrs & MPERM_W) && !(attrs & MPERM_R)),
"attrs must not define write permission without read "
"permission");
__ASSERT(!((attrs & MPERM_W) && (attrs & MPERM_X)),
"attrs must not define executable memory with write "
"permission");
/*
* The translation of the memory type / permissions / attributes
* flags in the attrs word to the TEX, C, B, S and AP bits of the
* target PTE is based on the reference manual:
* TEX, C, B, S: Table B3-10, chap. B3.8.2, p. B3-1363f.
* AP : Table B3-6, chap. B3.7.1, p. B3-1353.
* Device / strongly ordered memory is always assigned to a domain
* other than that used for normal memory. Assuming that userspace
* support utilizing the MMU is eventually implemented, a single
* modification of the DACR register when entering/leaving unprivi-
* leged mode could be used in order to enable/disable all device
* memory access without having to modify any PTs/PTEs.
*/
if (attrs & MT_STRONGLY_ORDERED) {
/* Strongly ordered is always shareable, S bit is ignored */
perms_attrs.tex = 0;
perms_attrs.cacheable = 0;
perms_attrs.bufferable = 0;
perms_attrs.shared = 0;
perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE;
} else if (attrs & MT_DEVICE) {
/*
* Shareability of device memory is determined by TEX, C, B.
* The S bit is ignored. C is always 0 for device memory.
*/
perms_attrs.shared = 0;
perms_attrs.cacheable = 0;
perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE;
if (attrs & MATTR_SHARED) {
perms_attrs.tex = 0;
perms_attrs.bufferable = 1;
} else {
perms_attrs.tex = 2;
perms_attrs.bufferable = 0;
}
} else if (attrs & MT_NORMAL) {
/*
* TEX[2] is always 1. TEX[1:0] contain the outer cache attri-
* butes encoding, C and B contain the inner cache attributes
* encoding.
*/
perms_attrs.tex |= ARM_MMU_TEX2_CACHEABLE_MEMORY;
perms_attrs.domain = ARM_MMU_DOMAIN_OS;
/* For normal memory, shareability depends on the S bit */
if (attrs & MATTR_SHARED) {
perms_attrs.shared = 1;
}
if (attrs & MATTR_CACHE_OUTER_WB_WA) {
perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_WA;
} else if (attrs & MATTR_CACHE_OUTER_WT_nWA) {
perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WT_nWA;
} else if (attrs & MATTR_CACHE_OUTER_WB_nWA) {
perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_nWA;
}
if (attrs & MATTR_CACHE_INNER_WB_WA) {
perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_WA;
perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_WA;
} else if (attrs & MATTR_CACHE_INNER_WT_nWA) {
perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WT_nWA;
perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WT_nWA;
} else if (attrs & MATTR_CACHE_INNER_WB_nWA) {
perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_nWA;
perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_nWA;
}
}
if (attrs & MATTR_NON_SECURE) {
perms_attrs.non_sec = 1;
}
if (attrs & MATTR_NON_GLOBAL) {
perms_attrs.not_global = 1;
}
/*
* Up next is the consideration of the case that a PTE shall be configured
* for a page that shall not be accessible at all (e.g. guard pages), and
* therefore has neither read nor write permissions. In the AP[2:1] access
* permission specification model, the only way to indicate this is to
* actually mask out the PTE's identifier bits, as otherwise, read permission
* is always granted for any valid PTE, it can't be revoked explicitly,
* unlike the write permission.
*/
if (!((attrs & MPERM_R) || (attrs & MPERM_W))) {
perms_attrs.id_mask = 0x0;
} else {
perms_attrs.id_mask = 0x3;
}
if (!(attrs & MPERM_W)) {
perms_attrs.acc_perms |= ARM_MMU_PERMS_AP2_DISABLE_WR;
}
if (attrs & MPERM_UNPRIVILEGED) {
perms_attrs.acc_perms |= ARM_MMU_PERMS_AP1_ENABLE_PL0;
}
if (!(attrs & MPERM_X)) {
perms_attrs.exec_never = 1;
}
return perms_attrs;
}
/**
* @brief Maps a 1 MB memory range via a level 1 page table entry
* Maps a 1 MB memory range using a level 1 page table entry of type
* 'section'. This type of entry saves a level 2 page table, but has
* two pre-conditions: the memory area to be mapped must contain at
* least 1 MB of contiguous memory, starting at an address with suit-
* able alignment. This mapping method should only be used for map-
* pings for which it is unlikely that the attributes of those mappings
* will mappings will change at run-time (e.g. code sections will al-
* ways be read-only and executable). Should the case occur that the
* permissions or attributes of a subset of a 1 MB section entry shall
* be re-configured at run-time, a L1 section entry will be broken
* down into 4k segments using a L2 table with identical attributes
* before any modifications are performed for the subset of the affec-
* ted 1 MB range. This comes with an undeterministic performance
* penalty at the time of re-configuration, therefore, any mappings
* for which L1 section entries are a valid option, shall be marked in
* their declaration with the MATTR_MAY_MAP_L1_SECTION flag.
*
* @param va 32-bit target virtual address to be mapped.
* @param pa 32-bit physical address to be mapped.
* @param perms_attrs Permission and attribute bits in the format
* used in the MMU's L1 page table entries.
*/
static void arm_mmu_l1_map_section(uint32_t va, uint32_t pa,
struct arm_mmu_perms_attrs perms_attrs)
{
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
__ASSERT(l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID,
"Unexpected non-zero L1 PTE ID %u for VA 0x%08X / PA 0x%08X",
l1_page_table.entries[l1_index].undefined.id,
va, pa);
l1_page_table.entries[l1_index].l1_section_1m.id =
(ARM_MMU_PTE_ID_SECTION & perms_attrs.id_mask);
l1_page_table.entries[l1_index].l1_section_1m.bufferable = perms_attrs.bufferable;
l1_page_table.entries[l1_index].l1_section_1m.cacheable = perms_attrs.cacheable;
l1_page_table.entries[l1_index].l1_section_1m.exec_never = perms_attrs.exec_never;
l1_page_table.entries[l1_index].l1_section_1m.domain = perms_attrs.domain;
l1_page_table.entries[l1_index].l1_section_1m.impl_def = 0;
l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 =
((perms_attrs.acc_perms & 0x1) << 1) | 0x1;
l1_page_table.entries[l1_index].l1_section_1m.tex = perms_attrs.tex;
l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 =
(perms_attrs.acc_perms >> 1) & 0x1;
l1_page_table.entries[l1_index].l1_section_1m.shared = perms_attrs.shared;
l1_page_table.entries[l1_index].l1_section_1m.not_global = perms_attrs.not_global;
l1_page_table.entries[l1_index].l1_section_1m.zero = 0;
l1_page_table.entries[l1_index].l1_section_1m.non_sec = perms_attrs.non_sec;
l1_page_table.entries[l1_index].l1_section_1m.base_address =
(pa >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT);
}
/**
* @brief Converts a L1 1 MB section mapping to a full L2 table
* When this function is called, something has happened that shouldn't
* happen for the sake of run-time performance and determinism: the
* attributes and/or permissions of a subset of a 1 MB memory range
* currently represented by a level 1 page table entry of type 'section'
* shall be modified so that they differ from the rest of the 1 MB
* range's attributes/permissions. Therefore, the single L1 page table
* entry has to be broken down to the full 256 4k-wide entries of a
* L2 page table with identical properties so that afterwards, the
* modification of the subset can be performed with a 4k granularity.
* The risk at this point is that all L2 tables are already in use,
* which will result in an assertion failure in the first contained
* #arm_mmu_l2_map_page() call.
* @warning While the conversion is being performed, interrupts are
* locked globally and the MMU is disabled (the required
* Zephyr code & data are still accessible in this state as
* those are identity mapped). Expect non-deterministic be-
* haviour / interrupt latencies while the conversion is in
* progress!
*
* @param va 32-bit virtual address within the 1 MB range that shall
* be converted from L1 1 MB section mapping to L2 4 kB page
* mappings.
* @param l2_page_table Pointer to an empty L2 page table allocated
* for the purpose of replacing the L1 section
* mapping.
*/
static void arm_mmu_remap_l1_section_to_l2_table(uint32_t va,
struct arm_mmu_l2_page_table *l2_page_table)
{
struct arm_mmu_perms_attrs perms_attrs = {0};
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t rem_size = MB(1);
uint32_t reg_val;
int lock_key;
/*
* Extract the permissions and attributes from the current 1 MB section entry.
* This data will be carried over to the resulting L2 page table.
*/
perms_attrs.acc_perms = (l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 << 1) |
((l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 >> 1) & 0x1);
perms_attrs.bufferable = l1_page_table.entries[l1_index].l1_section_1m.bufferable;
perms_attrs.cacheable = l1_page_table.entries[l1_index].l1_section_1m.cacheable;
perms_attrs.domain = l1_page_table.entries[l1_index].l1_section_1m.domain;
perms_attrs.id_mask = (l1_page_table.entries[l1_index].l1_section_1m.id ==
ARM_MMU_PTE_ID_INVALID) ? 0x0 : 0x3;
perms_attrs.not_global = l1_page_table.entries[l1_index].l1_section_1m.not_global;
perms_attrs.non_sec = l1_page_table.entries[l1_index].l1_section_1m.non_sec;
perms_attrs.shared = l1_page_table.entries[l1_index].l1_section_1m.shared;
perms_attrs.tex = l1_page_table.entries[l1_index].l1_section_1m.tex;
perms_attrs.exec_never = l1_page_table.entries[l1_index].l1_section_1m.exec_never;
/*
* Disable interrupts - no interrupts shall occur before the L2 table has
* been set up in place of the former L1 section entry.
*/
lock_key = arch_irq_lock();
/*
* Disable the MMU. The L1 PTE array and the L2 PT array may actually be
* covered by the L1 PTE we're about to replace, so access to this data
* must remain functional during the entire remap process. Yet, the only
* memory areas for which L1 1 MB section entries are even considered are
* those belonging to the Zephyr image. Those areas are *always* identity
* mapped, so the MMU can be turned off and the relevant data will still
* be available.
*/
reg_val = __get_SCTLR();
__set_SCTLR(reg_val & (~ARM_MMU_SCTLR_MMU_ENABLE_BIT));
/*
* Clear the entire L1 PTE & re-configure it as a L2 PT reference
* -> already sets the correct values for: zero0, zero1, impl_def.
*/
l1_page_table.entries[l1_index].word = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT;
l1_page_table.entries[l1_index].l2_page_table_ref.domain = perms_attrs.domain;
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = perms_attrs.non_sec;
l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address =
(((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) &
ARM_MMU_PT_L2_ADDR_MASK);
/* Align the target VA to the base address of the section we're converting */
va &= ~(MB(1) - 1);
while (rem_size > 0) {
arm_mmu_l2_map_page(va, va, perms_attrs);
rem_size -= KB(4);
va += KB(4);
}
/* Remap complete, re-enable the MMU, unlock the interrupts. */
invalidate_tlb_all();
__set_SCTLR(reg_val);
arch_irq_unlock(lock_key);
}
/**
* @brief Maps a 4 kB memory page using a L2 page table entry
* Maps a single 4 kB page of memory from the specified physical
* address to the specified virtual address, using the provided
* attributes and permissions which have already been converted
* from the system's format provided to arch_mem_map() to the
* bits / bit masks used in the L2 page table entry.
*
* @param va 32-bit target virtual address.
* @param pa 32-bit physical address.
* @param perms_attrs Permission and attribute bits in the format
* used in the MMU's L2 page table entries.
*/
static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa,
struct arm_mmu_perms_attrs perms_attrs)
{
struct arm_mmu_l2_page_table *l2_page_table = NULL;
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L2_INDEX_MASK;
/*
* Use the calculated L1 index in order to determine if a L2 page
* table is required in order to complete the current mapping.
* -> See below for an explanation of the possible scenarios.
*/
if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID ||
(l1_page_table.entries[l1_index].undefined.id & ARM_MMU_PTE_ID_SECTION) != 0) {
l2_page_table = arm_mmu_assign_l2_table(pa);
__ASSERT(l2_page_table != NULL,
"Unexpected L2 page table NULL pointer for VA 0x%08X",
va);
}
/*
* Check what is currently present at the corresponding L1 table entry.
* The following scenarios are possible:
* 1) The L1 PTE's ID bits are zero, as is the rest of the entry.
* In this case, the L1 PTE is currently unused. A new L2 PT to
* refer to in this entry has already been allocated above.
* 2) The L1 PTE's ID bits indicate a L2 PT reference entry (01).
* The corresponding L2 PT's address will be resolved using this
* entry.
* 3) The L1 PTE's ID bits may or may not be zero, and the rest of
* the descriptor contains some non-zero data. This always indicates
* an existing 1 MB section entry in this place. Checking only the
* ID bits wouldn't be enough, as the only way to indicate a section
* with neither R nor W permissions is to set the ID bits to 00 in
* the AP[2:1] permissions model. As we're now about to map a single
* page overlapping with the 1 MB section, the section has to be
* converted into a L2 table. Afterwards, the current page mapping
* can be added/modified.
*/
if (l1_page_table.entries[l1_index].word == 0) {
/* The matching L1 PT entry is currently unused */
l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT;
l1_page_table.entries[l1_index].l2_page_table_ref.zero0 = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.zero1 = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.impl_def = 0;
l1_page_table.entries[l1_index].l2_page_table_ref.domain = 0; /* TODO */
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec =
perms_attrs.non_sec;
l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address =
(((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) &
ARM_MMU_PT_L2_ADDR_MASK);
} else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) {
/* The matching L1 PT entry already points to a L2 PT */
l2_page_table = (struct arm_mmu_l2_page_table *)
((l1_page_table.entries[l1_index].word &
(ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT)));
/*
* The only configuration bit contained in the L2 PT entry is the
* NS bit. Set it according to the attributes passed to this function,
* warn if there is a mismatch between the current page's NS attribute
* value and the value currently contained in the L2 PT entry.
*/
if (l1_page_table.entries[l1_index].l2_page_table_ref.non_sec !=
perms_attrs.non_sec) {
LOG_WRN("NS bit mismatch in L2 PT reference at L1 index [%u], "
"re-configuring from %u to %u",
l1_index,
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec,
perms_attrs.non_sec);
l1_page_table.entries[l1_index].l2_page_table_ref.non_sec =
perms_attrs.non_sec;
}
} else if (l1_page_table.entries[l1_index].undefined.reserved != 0) {
/*
* The matching L1 PT entry currently holds a 1 MB section entry
* in order to save a L2 table (as it's neither completely blank
* nor a L2 PT reference), but now we have to map an overlapping
* 4 kB page, so the section entry must be converted to a L2 table
* first before the individual L2 entry for the page to be mapped is
* accessed. A blank L2 PT has already been assigned above.
*/
arm_mmu_remap_l1_section_to_l2_table(va, l2_page_table);
}
/*
* If the matching L2 PTE is blank, increment the number of used entries
* in the L2 table. If the L2 PTE already contains some data, we're re-
* placing the entry's data instead, the used entry count remains unchanged.
* Once again, checking the ID bits might be misleading if the PTE declares
* a page which has neither R nor W permissions.
*/
if (l2_page_table->entries[l2_index].word == 0) {
arm_mmu_inc_l2_table_entries(l2_page_table);
}
l2_page_table->entries[l2_index].l2_page_4k.id =
(ARM_MMU_PTE_ID_SMALL_PAGE & perms_attrs.id_mask);
l2_page_table->entries[l2_index].l2_page_4k.id |= perms_attrs.exec_never; /* XN in [0] */
l2_page_table->entries[l2_index].l2_page_4k.bufferable = perms_attrs.bufferable;
l2_page_table->entries[l2_index].l2_page_4k.cacheable = perms_attrs.cacheable;
l2_page_table->entries[l2_index].l2_page_4k.acc_perms10 =
((perms_attrs.acc_perms & 0x1) << 1) | 0x1;
l2_page_table->entries[l2_index].l2_page_4k.tex = perms_attrs.tex;
l2_page_table->entries[l2_index].l2_page_4k.acc_perms2 =
((perms_attrs.acc_perms >> 1) & 0x1);
l2_page_table->entries[l2_index].l2_page_4k.shared = perms_attrs.shared;
l2_page_table->entries[l2_index].l2_page_4k.not_global = perms_attrs.not_global;
l2_page_table->entries[l2_index].l2_page_4k.pa_base =
((pa >> ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT) &
ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_MASK);
}
/**
* @brief Unmaps a 4 kB memory page by clearing its L2 page table entry
* Unmaps a single 4 kB page of memory from the specified virtual
* address by clearing its respective L2 page table entry.
*
* @param va 32-bit virtual address to be unmapped.
*/
static void arm_mmu_l2_unmap_page(uint32_t va)
{
struct arm_mmu_l2_page_table *l2_page_table;
uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L2_INDEX_MASK;
if (l1_page_table.entries[l1_index].undefined.id != ARM_MMU_PTE_ID_L2_PT) {
/*
* No L2 PT currently exists for the given VA - this should be
* tolerated without an error, just as in the case that while
* a L2 PT exists, the corresponding PTE is blank - see explanation
* below, the same applies here.
*/
return;
}
l2_page_table = (struct arm_mmu_l2_page_table *)
((l1_page_table.entries[l1_index].word &
(ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT)));
if (l2_page_table->entries[l2_index].word == 0) {
/*
* We're supposed to unmap a page at the given VA, but there currently
* isn't anything mapped at this address, the L2 PTE is blank.
* -> This is normal if a memory area is being mapped via k_mem_map,
* which contains two calls to arch_mem_unmap (which effectively end up
* here) in order to unmap the leading and trailing guard pages.
* Therefore, it has to be expected that unmap calls are made for unmapped
* memory which hasn't been in use before.
* -> Just return, don't decrement the entry counter of the corresponding
* L2 page table, as we're not actually clearing any PTEs.
*/
return;
}
if ((l2_page_table->entries[l2_index].undefined.id & ARM_MMU_PTE_ID_SMALL_PAGE) !=
ARM_MMU_PTE_ID_SMALL_PAGE) {
LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid "
"page table entry type in level 2 page table at "
"L1 index [%u], L2 index [%u]", va, l1_index, l2_index);
return;
}
l2_page_table->entries[l2_index].word = 0;
arm_mmu_dec_l2_table_entries(l2_page_table);
}
/**
* @brief MMU boot-time initialization function
* Initializes the MMU at boot time. Sets up the page tables and
* applies any specified memory mappings for either the different
* sections of the Zephyr binary image, or for device memory as
* specified at the SoC level.
*
* @retval Always 0, errors are handled by assertions.
*/
int z_arm_mmu_init(void)
{
uint32_t mem_range;
uint32_t pa;
uint32_t va;
uint32_t attrs;
uint32_t pt_attrs = 0;
uint32_t rem_size;
uint32_t reg_val = 0;
struct arm_mmu_perms_attrs perms_attrs;
__ASSERT(KB(4) == CONFIG_MMU_PAGE_SIZE,
"MMU_PAGE_SIZE value %u is invalid, only 4 kB pages are supported\n",
CONFIG_MMU_PAGE_SIZE);
/* Set up the memory regions pre-defined by the image */
for (mem_range = 0; mem_range < ARRAY_SIZE(mmu_zephyr_ranges); mem_range++) {
pa = mmu_zephyr_ranges[mem_range].start;
rem_size = mmu_zephyr_ranges[mem_range].end - pa;
attrs = mmu_zephyr_ranges[mem_range].attrs;
perms_attrs = arm_mmu_convert_attr_flags(attrs);
/*
* Check if the L1 page table is within the region currently
* being mapped. If so, store the permissions and attributes
* of the current section. This information is required when
* writing to the TTBR0 register.
*/
if (((uint32_t)&l1_page_table >= pa) &&
((uint32_t)&l1_page_table < (pa + rem_size))) {
pt_attrs = attrs;
}
while (rem_size > 0) {
if (rem_size >= MB(1) && (pa & 0xFFFFF) == 0 &&
(attrs & MATTR_MAY_MAP_L1_SECTION)) {
/*
* Remaining area size > 1 MB & matching alignment
* -> map a 1 MB section instead of individual 4 kB
* pages with identical configuration.
*/
arm_mmu_l1_map_section(pa, pa, perms_attrs);
rem_size -= MB(1);
pa += MB(1);
} else {
arm_mmu_l2_map_page(pa, pa, perms_attrs);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
pa += KB(4);
}
}
}
/* Set up the memory regions defined at the SoC level */
for (mem_range = 0; mem_range < mmu_config.num_regions; mem_range++) {
pa = (uint32_t)(mmu_config.mmu_regions[mem_range].base_pa);
va = (uint32_t)(mmu_config.mmu_regions[mem_range].base_va);
rem_size = (uint32_t)(mmu_config.mmu_regions[mem_range].size);
attrs = mmu_config.mmu_regions[mem_range].attrs;
perms_attrs = arm_mmu_convert_attr_flags(attrs);
while (rem_size > 0) {
arm_mmu_l2_map_page(va, pa, perms_attrs);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
pa += KB(4);
}
}
/* Clear TTBR1 */
__asm__ __volatile__("mcr p15, 0, %0, c2, c0, 1" : : "r"(reg_val));
/* Write TTBCR: EAE, security not yet relevant, N[2:0] = 0 */
__asm__ __volatile__("mcr p15, 0, %0, c2, c0, 2"
: : "r"(reg_val));
/* Write TTBR0 */
reg_val = ((uint32_t)&l1_page_table.entries[0] & ~0x3FFF);
/*
* Set IRGN, RGN, S in TTBR0 based on the configuration of the
* memory area the actual page tables are located in.
*/
if (pt_attrs & MATTR_SHARED) {
reg_val |= ARM_MMU_TTBR_SHAREABLE_BIT;
}
if (pt_attrs & MATTR_CACHE_OUTER_WB_WA) {
reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_WA_CACHEABLE <<
ARM_MMU_TTBR_RGN_SHIFT);
} else if (pt_attrs & MATTR_CACHE_OUTER_WT_nWA) {
reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WT_CACHEABLE <<
ARM_MMU_TTBR_RGN_SHIFT);
} else if (pt_attrs & MATTR_CACHE_OUTER_WB_nWA) {
reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_nWA_CACHEABLE <<
ARM_MMU_TTBR_RGN_SHIFT);
}
if (pt_attrs & MATTR_CACHE_INNER_WB_WA) {
reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY;
} else if (pt_attrs & MATTR_CACHE_INNER_WT_nWA) {
reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY;
} else if (pt_attrs & MATTR_CACHE_INNER_WB_nWA) {
reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY;
reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY;
}
__set_TTBR0(reg_val);
/* Write DACR -> all domains to client = 01b. */
reg_val = ARM_MMU_DACR_ALL_DOMAINS_CLIENT;
__set_DACR(reg_val);
invalidate_tlb_all();
/* Enable the MMU and Cache in SCTLR */
reg_val = __get_SCTLR();
reg_val |= ARM_MMU_SCTLR_AFE_BIT;
reg_val |= ARM_MMU_SCTLR_ICACHE_ENABLE_BIT;
reg_val |= ARM_MMU_SCTLR_DCACHE_ENABLE_BIT;
reg_val |= ARM_MMU_SCTLR_MMU_ENABLE_BIT;
__set_SCTLR(reg_val);
return 0;
}
/**
* @brief ARMv7-specific implementation of memory mapping at run-time
* Maps memory according to the parameters provided by the caller
* at run-time.
*
* @param virt 32-bit target virtual address.
* @param phys 32-bit physical address.
* @param size Size (in bytes) of the memory area to map.
* @param flags Memory attributes & permissions. Comp. K_MEM_...
* flags in kernel/mm.h.
* @retval 0 on success, -EINVAL if an invalid parameter is detected.
*/
static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
{
uint32_t va = (uint32_t)virt;
uint32_t pa = (uint32_t)phys;
uint32_t rem_size = (uint32_t)size;
uint32_t conv_flags = MPERM_R;
struct arm_mmu_perms_attrs perms_attrs;
int key;
if (size == 0) {
LOG_ERR("Cannot map physical memory at 0x%08X: invalid "
"zero size", (uint32_t)phys);
return -EINVAL;
}
switch (flags & K_MEM_CACHE_MASK) {
case K_MEM_CACHE_NONE:
default:
conv_flags |= MT_DEVICE;
break;
case K_MEM_CACHE_WB:
conv_flags |= MT_NORMAL;
conv_flags |= MATTR_SHARED;
if (flags & K_MEM_PERM_RW) {
conv_flags |= MATTR_CACHE_OUTER_WB_WA;
conv_flags |= MATTR_CACHE_INNER_WB_WA;
} else {
conv_flags |= MATTR_CACHE_OUTER_WB_nWA;
conv_flags |= MATTR_CACHE_INNER_WB_nWA;
}
break;
case K_MEM_CACHE_WT:
conv_flags |= MT_NORMAL;
conv_flags |= MATTR_SHARED;
conv_flags |= MATTR_CACHE_OUTER_WT_nWA;
conv_flags |= MATTR_CACHE_INNER_WT_nWA;
break;
}
if (flags & K_MEM_PERM_RW) {
conv_flags |= MPERM_W;
}
if (flags & K_MEM_PERM_EXEC) {
conv_flags |= MPERM_X;
}
perms_attrs = arm_mmu_convert_attr_flags(conv_flags);
key = arch_irq_lock();
while (rem_size > 0) {
arm_mmu_l2_map_page(va, pa, perms_attrs);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
pa += KB(4);
}
arch_irq_unlock(key);
return 0;
}
/**
* @brief Arch-specific wrapper function for memory mapping at run-time
* Maps memory according to the parameters provided by the caller
* at run-time. This function wraps the ARMv7 MMU specific implementation
* #__arch_mem_map() for the upper layers of the memory management.
* If the map operation fails, a kernel panic will be triggered.
*
* @param virt 32-bit target virtual address.
* @param phys 32-bit physical address.
* @param size Size (in bytes) of the memory area to map.
* @param flags Memory attributes & permissions. Comp. K_MEM_...
* flags in kernel/mm.h.
*/
void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags)
{
int ret = __arch_mem_map(virt, phys, size, flags);
if (ret) {
LOG_ERR("__arch_mem_map() returned %d", ret);
k_panic();
} else {
invalidate_tlb_all();
}
}
/**
* @brief ARMv7-specific implementation of memory unmapping at run-time
* Unmaps memory according to the parameters provided by the caller
* at run-time.
*
* @param addr 32-bit virtual address to unmap.
* @param size Size (in bytes) of the memory area to unmap.
* @retval 0 on success, -EINVAL if an invalid parameter is detected.
*/
static int __arch_mem_unmap(void *addr, size_t size)
{
uint32_t va = (uint32_t)addr;
uint32_t rem_size = (uint32_t)size;
int key;
if (addr == NULL) {
LOG_ERR("Cannot unmap virtual memory: invalid NULL pointer");
return -EINVAL;
}
if (size == 0) {
LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid "
"zero size", (uint32_t)addr);
return -EINVAL;
}
key = arch_irq_lock();
while (rem_size > 0) {
arm_mmu_l2_unmap_page(va);
rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size;
va += KB(4);
}
arch_irq_unlock(key);
return 0;
}
/**
* @brief Arch-specific wrapper function for memory unmapping at run-time
* Unmaps memory according to the parameters provided by the caller
* at run-time. This function wraps the ARMv7 MMU specific implementation
* #__arch_mem_unmap() for the upper layers of the memory management.
*
* @param addr 32-bit virtual address to unmap.
* @param size Size (in bytes) of the memory area to unmap.
*/
void arch_mem_unmap(void *addr, size_t size)
{
int ret = __arch_mem_unmap(addr, size);
if (ret) {
LOG_ERR("__arch_mem_unmap() returned %d", ret);
} else {
invalidate_tlb_all();
}
}
/**
* @brief Arch-specific virtual-to-physical address resolver function
* ARMv7 MMU specific implementation of a function that resolves the
* physical address corresponding to the given virtual address.
*
* @param virt 32-bit target virtual address to resolve.
* @param phys Pointer to a variable to which the resolved physical
* address will be written. May be NULL if this information
* is not actually required by the caller.
* @retval 0 if the physical address corresponding to the specified
* virtual address could be resolved successfully, -EFAULT
* if the specified virtual address is not currently mapped.
*/
int arch_page_phys_get(void *virt, uintptr_t *phys)
{
uint32_t l1_index = ((uint32_t)virt >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L1_INDEX_MASK;
uint32_t l2_index = ((uint32_t)virt >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) &
ARM_MMU_PTE_L2_INDEX_MASK;
struct arm_mmu_l2_page_table *l2_page_table;
uint32_t pa_resolved = 0;
uint32_t l2_pt_resolved;
int rc = 0;
int key;
key = arch_irq_lock();
if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_SECTION) {
/*
* If the virtual address points to a level 1 PTE whose ID bits
* identify it as a 1 MB section entry rather than a level 2 PT
* entry, the given VA belongs to a memory region used by the
* Zephyr image itself - it is only for those static regions that
* L1 Section entries are used to save L2 tables if a sufficient-
* ly large block of memory is specified. The memory regions be-
* longing to the Zephyr image are identity mapped -> just return
* the value of the VA as the value of the PA.
*/
pa_resolved = (uint32_t)virt;
} else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) {
/*
* The VA points to a level 1 PTE which re-directs to a level 2
* PT. -> Assemble the level 2 PT pointer and resolve the PA for
* the specified VA from there.
*/
l2_pt_resolved =
l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address;
l2_pt_resolved <<= ARM_MMU_PT_L2_ADDR_SHIFT;
l2_page_table = (struct arm_mmu_l2_page_table *)l2_pt_resolved;
/*
* Check if the PTE for the specified VA is actually in use before
* assembling & returning the corresponding PA. k_mem_unmap will
* call this function for the leading & trailing guard pages when
* unmapping a VA. As those guard pages were explicitly unmapped
* when the VA was originally mapped, their L2 PTEs will be empty.
* In that case, the return code of this function must not be 0.
*/
if (l2_page_table->entries[l2_index].word == 0) {
rc = -EFAULT;
}
pa_resolved = l2_page_table->entries[l2_index].l2_page_4k.pa_base;
pa_resolved <<= ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT;
pa_resolved |= ((uint32_t)virt & ARM_MMU_ADDR_BELOW_PAGE_GRAN_MASK);
} else {
/* The level 1 PTE is invalid -> the specified VA is not mapped */
rc = -EFAULT;
}
arch_irq_unlock(key);
if (phys) {
*phys = (uintptr_t)pa_resolved;
}
return rc;
}
View git blame Copy permalink