From 70c403c2156d7f8971c676b5fbefe15db8e7257f Mon Sep 17 00:00:00 2001 From: Immo Birnbaum Date: Thu, 15 Jul 2021 10:33:58 +0200 Subject: [PATCH] arch: arm: core: aarch32: introduce basic ARMv7 MMU support An initial implementation for memory management using the ARMv7 MMU. A single L1 translation table for the whole 4 GB address space is al- ways present, a configurable number of L2 page tables are linked to the L1 table based on the static memory area configuration at boot time, or whenever arch_mem_map/arch_mem_unmap are called at run-time. Currently, a CPU with the Multiprocessor Extensions and execution at PL1 are always assumed. Userspace-related features or thread stack guard pages are not yet supported. Neither are LPAE, PXN or TEX re- mapping. All mappings are currently assigned to the same domain. Re- garding the permissions model, access permissions are specified using the AP[2:1] model rather than the older AP[2:0] model, which, accor- ding to ARM's documentation, is deprecated and should no longer be used. The newer model adds some complexity when it comes to mapping pages as unaccessible (the AP[2:1] model doesn't support explicit specification of "no R, no W" permissions, it's always at least "RO"), this is accomplished by invalidating the ID bits of the respective page's PTE. Includes sources, Kconfig integration, adjusted CMakeLists and the modified linker command file (proper section alignment!). Signed-off-by: Immo Birnbaum --- arch/arm/core/aarch32/CMakeLists.txt | 1 + arch/arm/core/aarch32/Kconfig | 1 + arch/arm/core/aarch32/mmu/CMakeLists.txt | 5 + arch/arm/core/aarch32/mmu/Kconfig | 35 + arch/arm/core/aarch32/mmu/arm_mmu.c | 1094 +++++++++++++++++ arch/arm/core/aarch32/mmu/arm_mmu_priv.h | 205 +++ arch/arm/include/aarch32/kernel_arch_func.h | 8 +- include/arch/arm/aarch32/arch.h | 5 + .../arm/aarch32/cortex_a_r/scripts/linker.ld | 18 +- include/arch/arm/aarch32/mmu/arm_mmu.h | 105 ++ 10 files changed, 1475 insertions(+), 2 deletions(-) create mode 100644 arch/arm/core/aarch32/mmu/CMakeLists.txt create mode 100644 arch/arm/core/aarch32/mmu/Kconfig create mode 100644 arch/arm/core/aarch32/mmu/arm_mmu.c create mode 100644 arch/arm/core/aarch32/mmu/arm_mmu_priv.h create mode 100644 include/arch/arm/aarch32/mmu/arm_mmu.h diff --git a/arch/arm/core/aarch32/CMakeLists.txt b/arch/arm/core/aarch32/CMakeLists.txt index 5ae7d1d0d07..2a51c5dcbbf 100644 --- a/arch/arm/core/aarch32/CMakeLists.txt +++ b/arch/arm/core/aarch32/CMakeLists.txt @@ -27,6 +27,7 @@ add_subdirectory_ifdef(CONFIG_ARM_SECURE_FIRMWARE cortex_m/tz) add_subdirectory_ifdef(CONFIG_ARM_NONSECURE_FIRMWARE cortex_m/tz) add_subdirectory_ifdef(CONFIG_ARM_MPU mpu) +add_subdirectory_ifdef(CONFIG_ARM_AARCH32_MMU mmu) add_subdirectory_ifdef(CONFIG_CPU_CORTEX_R cortex_a_r) diff --git a/arch/arm/core/aarch32/Kconfig b/arch/arm/core/aarch32/Kconfig index 38dfab0d0e6..7bce4abaf50 100644 --- a/arch/arm/core/aarch32/Kconfig +++ b/arch/arm/core/aarch32/Kconfig @@ -306,3 +306,4 @@ rsource "cortex_m/Kconfig" rsource "cortex_a_r/Kconfig" rsource "mpu/Kconfig" +rsource "mmu/Kconfig" diff --git a/arch/arm/core/aarch32/mmu/CMakeLists.txt b/arch/arm/core/aarch32/mmu/CMakeLists.txt new file mode 100644 index 00000000000..5356b078bc3 --- /dev/null +++ b/arch/arm/core/aarch32/mmu/CMakeLists.txt @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 + +zephyr_library() + +zephyr_library_sources(arm_mmu.c) diff --git a/arch/arm/core/aarch32/mmu/Kconfig b/arch/arm/core/aarch32/mmu/Kconfig new file mode 100644 index 00000000000..db523744868 --- /dev/null +++ b/arch/arm/core/aarch32/mmu/Kconfig @@ -0,0 +1,35 @@ +# +# ARMv7 Memory Management Unit (MMU) configuration options +# +# Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG +# SPDX-License-Identifier: Apache-2.0 +# + +if CPU_HAS_MMU + +config ARM_AARCH32_MMU + bool "ARMv7 Cortex-A MMU Support" + default y if CPU_AARCH32_CORTEX_A + select MMU + select SRAM_REGION_PERMISSIONS + select THREAD_STACK_INFO + select ARCH_HAS_EXECUTABLE_PAGE_BIT + help + The current CPU has an ARMv7 Memory Management Unit. + +config ARM_MMU_NUM_L2_TABLES + depends on ARM_AARCH32_MMU + int "Number of L2 translation tables available to the MMU" + default 64 + help + Number of level 2 translation tables. Each level 2 table + covers 1 MB of address space. + +config ARM_MMU_REGION_MIN_ALIGN_AND_SIZE + int + default 4096 + help + Minimum size (and alignment) of an ARM MMU page. + This value should not be modified. + +endif # CPU_HAS_MMU diff --git a/arch/arm/core/aarch32/mmu/arm_mmu.c b/arch/arm/core/aarch32/mmu/arm_mmu.c new file mode 100644 index 00000000000..9167a17b0bf --- /dev/null +++ b/arch/arm/core/aarch32/mmu/arm_mmu.c @@ -0,0 +1,1094 @@ +/* + * ARMv7 MMU support + * + * This implementation supports the Short-descriptor translation + * table format. The standard page size is 4 kB, 1 MB sections + * are only used for mapping the code and data of the Zephyr image. + * Secure mode and PL1 is always assumed. LPAE and PXN extensions + * as well as TEX remapping are not supported. The AP[2:1] plus + * Access flag permissions model is used, as the AP[2:0] model is + * deprecated. As the AP[2:1] model can only disable write access, + * the read permission flag is always implied. + * + * Reference documentation: + * ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition, + * ARM document ID DDI0406C Rev. d, March 2018 + * + * Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include "arm_mmu_priv.h" + +LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); + +/* Level 1 page table: always required, must be 16k-aligned */ +static struct arm_mmu_l1_page_table + l1_page_table __aligned(KB(16)) = {0}; +/* + * Array of level 2 page tables with 4k granularity: + * each table covers a range of 1 MB, the number of L2 tables + * is configurable. + */ +static struct arm_mmu_l2_page_table + l2_page_tables[CONFIG_ARM_MMU_NUM_L2_TABLES] __aligned(KB(1)) = {0}; +/* + * For each level 2 page table, a separate dataset tracks + * if the respective table is in use, if so, to which 1 MB + * virtual address range it is assigned, and how many entries, + * each mapping a 4 kB page, it currently contains. + */ +static struct arm_mmu_l2_page_table_status + l2_page_tables_status[CONFIG_ARM_MMU_NUM_L2_TABLES] = {0}; + +/* Available L2 tables count & next free index for an L2 table request */ +static uint32_t arm_mmu_l2_tables_free = CONFIG_ARM_MMU_NUM_L2_TABLES; +static uint32_t arm_mmu_l2_next_free_table; + +/* + * Static definition of all code & data memory regions of the + * current Zephyr image. This information must be available & + * processed upon MMU initialization. + */ +static const struct arm_mmu_flat_range mmu_zephyr_ranges[] = { + /* + * Mark the zephyr execution regions (data, bss, noinit, etc.) + * cacheable, read / write and non-executable + */ + { .name = "zephyr_data", + .start = (uint32_t)_image_ram_start, + .end = (uint32_t)_image_ram_end, + .attrs = MT_NORMAL | MATTR_SHARED | + MPERM_R | MPERM_W | + MATTR_CACHE_OUTER_WB_WA | MATTR_CACHE_INNER_WB_WA}, + + /* Mark text segment cacheable, read only and executable */ + { .name = "zephyr_code", + .start = (uint32_t)__text_region_start, + .end = (uint32_t)__text_region_end, + .attrs = MT_NORMAL | MATTR_SHARED | + MPERM_R | MPERM_X | + MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA | + MATTR_MAY_MAP_L1_SECTION}, + + /* Mark rodata segment cacheable, read only and non-executable */ + { .name = "zephyr_rodata", + .start = (uint32_t)__rodata_region_start, + .end = (uint32_t)__rodata_region_end, + .attrs = MT_NORMAL | MATTR_SHARED | + MPERM_R | + MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA | + MATTR_MAY_MAP_L1_SECTION}, +#ifdef CONFIG_NOCACHE_MEMORY + /* Mark nocache segment read / write and non-executable */ + { .name = "nocache", + .start = (uint32_t)_nocache_ram_start, + .end = (uint32_t)_nocache_ram_end, + .attrs = MT_STRONGLY_ORDERED | + MPERM_R | MPERM_W}, +#endif +}; + +static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa, + struct arm_mmu_perms_attrs perms_attrs); + +/** + * @brief Invalidates the TLB + * Helper function which invalidates the entire TLB. This action + * is performed whenever the MMU is (re-)enabled or changes to the + * page tables are made at run-time, as the TLB might contain entries + * which are no longer valid once the changes are applied. + */ +static void invalidate_tlb_all(void) +{ + __set_TLBIALL(0); /* 0 = opc2 = invalidate entire TLB */ + __DSB(); + __ISB(); +} + +/** + * @brief Returns a free level 2 page table + * Initializes and returns the next free L2 page table whenever + * a page is to be mapped in a 1 MB virtual address range that + * is not yet covered by a level 2 page table. + * + * @param va 32-bit virtual address to be mapped. + * @retval pointer to the L2 table now assigned to the 1 MB + * address range the target virtual address is in. + */ +static struct arm_mmu_l2_page_table *arm_mmu_assign_l2_table(uint32_t va) +{ + struct arm_mmu_l2_page_table *l2_page_table; + + __ASSERT(arm_mmu_l2_tables_free > 0, + "Cannot set up L2 page table for VA 0x%08X: " + "no more free L2 page tables available\n", + va); + __ASSERT(l2_page_tables_status[arm_mmu_l2_next_free_table].entries == 0, + "Cannot set up L2 page table for VA 0x%08X: " + "expected empty L2 table at index [%u], but the " + "entries value is %u\n", + va, arm_mmu_l2_next_free_table, + l2_page_tables_status[arm_mmu_l2_next_free_table].entries); + + /* + * Store in the status dataset of the L2 table to be returned + * which 1 MB virtual address range it is being assigned to. + * Set the current page table entry count to 0. + */ + l2_page_tables_status[arm_mmu_l2_next_free_table].l1_index = + ((va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK); + l2_page_tables_status[arm_mmu_l2_next_free_table].entries = 0; + l2_page_table = &l2_page_tables[arm_mmu_l2_next_free_table]; + + /* + * Decrement the available L2 page table count. As long as at + * least one more L2 table is available afterwards, update the + * L2 next free table index. If we're about to return the last + * available L2 table, calculating a next free table index is + * impossible. + */ + --arm_mmu_l2_tables_free; + if (arm_mmu_l2_tables_free > 0) { + do { + arm_mmu_l2_next_free_table = (arm_mmu_l2_next_free_table + 1) % + CONFIG_ARM_MMU_NUM_L2_TABLES; + } while (l2_page_tables_status[arm_mmu_l2_next_free_table].entries != 0); + } + + return l2_page_table; +} + +/** + * @brief Releases a level 2 page table + * Releases a level 2 page table, marking it as no longer in use. + * From that point on, it can be re-used for mappings in another + * 1 MB virtual address range. This function is called whenever + * it is determined during an unmap call at run-time that the page + * table entry count in the respective page table has reached 0. + * + * @param l2_page_table Pointer to L2 page table to be released. + */ +static void arm_mmu_release_l2_table(struct arm_mmu_l2_page_table *l2_page_table) +{ + uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table); + + l2_page_tables_status[l2_page_table_index].l1_index = 0; + if (arm_mmu_l2_tables_free == 0) { + arm_mmu_l2_next_free_table = l2_page_table_index; + } + ++arm_mmu_l2_tables_free; +} + +/** + * @brief Increments the page table entry counter of a L2 page table + * Increments the page table entry counter of a level 2 page table. + * Contains a check to ensure that no attempts are made to set up + * more page table entries than the table can hold. + * + * @param l2_page_table Pointer to the L2 page table whose entry + * counter shall be incremented. + */ +static void arm_mmu_inc_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table) +{ + uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table); + + __ASSERT(l2_page_tables_status[l2_page_table_index].entries < ARM_MMU_PT_L2_NUM_ENTRIES, + "Cannot increment entry count of the L2 page table at index " + "[%u] / addr %p / ref L1[%u]: maximum entry count already reached", + l2_page_table_index, l2_page_table, + l2_page_tables_status[l2_page_table_index].l1_index); + + ++l2_page_tables_status[l2_page_table_index].entries; +} + +/** + * @brief Decrements the page table entry counter of a L2 page table + * Decrements the page table entry counter of a level 2 page table. + * Contains a check to ensure that no attempts are made to remove + * entries from the respective table that aren't actually there. + * + * @param l2_page_table Pointer to the L2 page table whose entry + * counter shall be decremented. + */ +static void arm_mmu_dec_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table) +{ + uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table); + + __ASSERT(l2_page_tables_status[l2_page_table_index].entries > 0, + "Cannot decrement entry count of the L2 page table at index " + "[%u] / addr %p / ref L1[%u]: entry count is already zero", + l2_page_table_index, l2_page_table, + l2_page_tables_status[l2_page_table_index].l1_index); + + if (--l2_page_tables_status[l2_page_table_index].entries == 0) { + arm_mmu_release_l2_table(l2_page_table); + } +} + +/** + * @brief Converts memory attributes and permissions to MMU format + * Converts memory attributes and permissions as used in the boot- + * time memory mapping configuration data array (MT_..., MATTR_..., + * MPERM_...) to the equivalent bit (field) values used in the MMU's + * L1 and L2 page table entries. Contains plausibility checks. + * + * @param attrs type/attribute/permissions flags word obtained from + * an entry of the mmu_config mapping data array. + * @retval A struct containing the information from the input flags + * word converted to the bits / bit fields used in L1 and + * L2 page table entries. + */ +static struct arm_mmu_perms_attrs arm_mmu_convert_attr_flags(uint32_t attrs) +{ + struct arm_mmu_perms_attrs perms_attrs = {0}; + + __ASSERT(((attrs & MT_MASK) > 0), + "Cannot convert attrs word to PTE control bits: no " + "memory type specified"); + __ASSERT(!((attrs & MPERM_W) && !(attrs & MPERM_R)), + "attrs must not define write permission without read " + "permission"); + __ASSERT(!((attrs & MPERM_W) && (attrs & MPERM_X)), + "attrs must not define executable memory with write " + "permission"); + + /* + * The translation of the memory type / permissions / attributes + * flags in the attrs word to the TEX, C, B, S and AP bits of the + * target PTE is based on the reference manual: + * TEX, C, B, S: Table B3-10, chap. B3.8.2, p. B3-1363f. + * AP : Table B3-6, chap. B3.7.1, p. B3-1353. + * Device / strongly ordered memory is always assigned to a domain + * other than that used for normal memory. Assuming that userspace + * support utilizing the MMU is eventually implemented, a single + * modification of the DACR register when entering/leaving unprivi- + * leged mode could be used in order to enable/disable all device + * memory access without having to modify any PTs/PTEs. + */ + + if (attrs & MT_STRONGLY_ORDERED) { + /* Strongly ordered is always shareable, S bit is ignored */ + perms_attrs.tex = 0; + perms_attrs.cacheable = 0; + perms_attrs.bufferable = 0; + perms_attrs.shared = 0; + perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE; + } else if (attrs & MT_DEVICE) { + /* + * Shareability of device memory is determined by TEX, C, B. + * The S bit is ignored. C is always 0 for device memory. + */ + perms_attrs.shared = 0; + perms_attrs.cacheable = 0; + perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE; + + if (attrs & MATTR_SHARED) { + perms_attrs.tex = 0; + perms_attrs.bufferable = 1; + } else { + perms_attrs.tex = 2; + perms_attrs.bufferable = 0; + } + } else if (attrs & MT_NORMAL) { + /* + * TEX[2] is always 1. TEX[1:0] contain the outer cache attri- + * butes encoding, C and B contain the inner cache attributes + * encoding. + */ + perms_attrs.tex |= ARM_MMU_TEX2_CACHEABLE_MEMORY; + perms_attrs.domain = ARM_MMU_DOMAIN_OS; + + /* For normal memory, shareability depends on the S bit */ + if (attrs & MATTR_SHARED) { + perms_attrs.shared = 1; + } + + if (attrs & MATTR_CACHE_OUTER_WB_WA) { + perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_WA; + } else if (attrs & MATTR_CACHE_OUTER_WT_nWA) { + perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WT_nWA; + } else if (attrs & MATTR_CACHE_OUTER_WB_nWA) { + perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_nWA; + } + + if (attrs & MATTR_CACHE_INNER_WB_WA) { + perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_WA; + perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_WA; + } else if (attrs & MATTR_CACHE_INNER_WT_nWA) { + perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WT_nWA; + perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WT_nWA; + } else if (attrs & MATTR_CACHE_INNER_WB_nWA) { + perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_nWA; + perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_nWA; + } + } + + if (attrs & MATTR_NON_SECURE) { + perms_attrs.non_sec = 1; + } + if (attrs & MATTR_NON_GLOBAL) { + perms_attrs.not_global = 1; + } + + /* + * Up next is the consideration of the case that a PTE shall be configured + * for a page that shall not be accessible at all (e.g. guard pages), and + * therefore has neither read nor write permissions. In the AP[2:1] access + * permission specification model, the only way to indicate this is to + * actually mask out the PTE's identifier bits, as otherwise, read permission + * is always granted for any valid PTE, it can't be revoked explicitly, + * unlike the write permission. + */ + if (!((attrs & MPERM_R) || (attrs & MPERM_W))) { + perms_attrs.id_mask = 0x0; + } else { + perms_attrs.id_mask = 0x3; + } + if (!(attrs & MPERM_W)) { + perms_attrs.acc_perms |= ARM_MMU_PERMS_AP2_DISABLE_WR; + } + if (attrs & MPERM_UNPRIVILEGED) { + perms_attrs.acc_perms |= ARM_MMU_PERMS_AP1_ENABLE_PL0; + } + if (!(attrs & MPERM_X)) { + perms_attrs.exec_never = 1; + } + + return perms_attrs; +} + +/** + * @brief Maps a 1 MB memory range via a level 1 page table entry + * Maps a 1 MB memory range using a level 1 page table entry of type + * 'section'. This type of entry saves a level 2 page table, but has + * two pre-conditions: the memory area to be mapped must contain at + * least 1 MB of contiguous memory, starting at an address with suit- + * able alignment. This mapping method should only be used for map- + * pings for which it is unlikely that the attributes of those mappings + * will mappings will change at run-time (e.g. code sections will al- + * ways be read-only and executable). Should the case occur that the + * permissions or attributes of a subset of a 1 MB section entry shall + * be re-configured at run-time, a L1 section entry will be broken + * down into 4k segments using a L2 table with identical attributes + * before any modifications are performed for the subset of the affec- + * ted 1 MB range. This comes with an undeterministic performance + * penalty at the time of re-configuration, therefore, any mappings + * for which L1 section entries are a valid option, shall be marked in + * their declaration with the MATTR_MAY_MAP_L1_SECTION flag. + * + * @param va 32-bit target virtual address to be mapped. + * @param pa 32-bit physical address to be mapped. + * @param perms_attrs Permission and attribute bits in the format + * used in the MMU's L1 page table entries. + */ +static void arm_mmu_l1_map_section(uint32_t va, uint32_t pa, + struct arm_mmu_perms_attrs perms_attrs) +{ + uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L1_INDEX_MASK; + + __ASSERT(l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID, + "Unexpected non-zero L1 PTE ID %u for VA 0x%08X / PA 0x%08X", + l1_page_table.entries[l1_index].undefined.id, + va, pa); + + l1_page_table.entries[l1_index].l1_section_1m.id = + (ARM_MMU_PTE_ID_SECTION & perms_attrs.id_mask); + l1_page_table.entries[l1_index].l1_section_1m.bufferable = perms_attrs.bufferable; + l1_page_table.entries[l1_index].l1_section_1m.cacheable = perms_attrs.cacheable; + l1_page_table.entries[l1_index].l1_section_1m.exec_never = perms_attrs.exec_never; + l1_page_table.entries[l1_index].l1_section_1m.domain = perms_attrs.domain; + l1_page_table.entries[l1_index].l1_section_1m.impl_def = 0; + l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 = + ((perms_attrs.acc_perms & 0x1) << 1) | 0x1; + l1_page_table.entries[l1_index].l1_section_1m.tex = perms_attrs.tex; + l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 = + (perms_attrs.acc_perms >> 1) & 0x1; + l1_page_table.entries[l1_index].l1_section_1m.shared = perms_attrs.shared; + l1_page_table.entries[l1_index].l1_section_1m.not_global = perms_attrs.not_global; + l1_page_table.entries[l1_index].l1_section_1m.zero = 0; + l1_page_table.entries[l1_index].l1_section_1m.non_sec = perms_attrs.non_sec; + l1_page_table.entries[l1_index].l1_section_1m.base_address = + (pa >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT); +} + +/** + * @brief Converts a L1 1 MB section mapping to a full L2 table + * When this function is called, something has happened that shouldn't + * happen for the sake of run-time performance and determinism: the + * attributes and/or permissions of a subset of a 1 MB memory range + * currently represented by a level 1 page table entry of type 'section' + * shall be modified so that they differ from the rest of the 1 MB + * range's attributes/permissions. Therefore, the single L1 page table + * entry has to be broken down to the full 256 4k-wide entries of a + * L2 page table with identical properties so that afterwards, the + * modification of the subset can be performed with a 4k granularity. + * The risk at this point is that all L2 tables are already in use, + * which will result in an assertion failure in the first contained + * #arm_mmu_l2_map_page() call. + * @warning While the conversion is being performed, interrupts are + * locked globally and the MMU is disabled (the required + * Zephyr code & data are still accessible in this state as + * those are identity mapped). Expect non-deterministic be- + * haviour / interrupt latencies while the conversion is in + * progress! + * + * @param va 32-bit virtual address within the 1 MB range that shall + * be converted from L1 1 MB section mapping to L2 4 kB page + * mappings. + * @param l2_page_table Pointer to an empty L2 page table allocated + * for the purpose of replacing the L1 section + * mapping. + */ +static void arm_mmu_remap_l1_section_to_l2_table(uint32_t va, + struct arm_mmu_l2_page_table *l2_page_table) +{ + struct arm_mmu_perms_attrs perms_attrs = {0}; + uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L1_INDEX_MASK; + uint32_t rem_size = MB(1); + uint32_t reg_val; + int lock_key; + + /* + * Extract the permissions and attributes from the current 1 MB section entry. + * This data will be carried over to the resulting L2 page table. + */ + + perms_attrs.acc_perms = (l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 << 1) | + ((l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 >> 1) & 0x1); + perms_attrs.bufferable = l1_page_table.entries[l1_index].l1_section_1m.bufferable; + perms_attrs.cacheable = l1_page_table.entries[l1_index].l1_section_1m.cacheable; + perms_attrs.domain = l1_page_table.entries[l1_index].l1_section_1m.domain; + perms_attrs.id_mask = (l1_page_table.entries[l1_index].l1_section_1m.id == + ARM_MMU_PTE_ID_INVALID) ? 0x0 : 0x3; + perms_attrs.not_global = l1_page_table.entries[l1_index].l1_section_1m.not_global; + perms_attrs.non_sec = l1_page_table.entries[l1_index].l1_section_1m.non_sec; + perms_attrs.shared = l1_page_table.entries[l1_index].l1_section_1m.shared; + perms_attrs.tex = l1_page_table.entries[l1_index].l1_section_1m.tex; + perms_attrs.exec_never = l1_page_table.entries[l1_index].l1_section_1m.exec_never; + + /* + * Disable interrupts - no interrupts shall occur before the L2 table has + * been set up in place of the former L1 section entry. + */ + + lock_key = arch_irq_lock(); + + /* + * Disable the MMU. The L1 PTE array and the L2 PT array may actually be + * covered by the L1 PTE we're about to replace, so access to this data + * must remain functional during the entire remap process. Yet, the only + * memory areas for which L1 1 MB section entries are even considered are + * those belonging to the Zephyr image. Those areas are *always* identity + * mapped, so the MMU can be turned off and the relevant data will still + * be available. + */ + + reg_val = __get_SCTLR(); + __set_SCTLR(reg_val & (~ARM_MMU_SCTLR_MMU_ENABLE_BIT)); + + /* + * Clear the entire L1 PTE & re-configure it as a L2 PT reference + * -> already sets the correct values for: zero0, zero1, impl_def. + */ + l1_page_table.entries[l1_index].word = 0; + + l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT; + l1_page_table.entries[l1_index].l2_page_table_ref.domain = perms_attrs.domain; + l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = perms_attrs.non_sec; + l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address = + (((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) & + ARM_MMU_PT_L2_ADDR_MASK); + + /* Align the target VA to the base address of the section we're converting */ + va &= ~(MB(1) - 1); + while (rem_size > 0) { + arm_mmu_l2_map_page(va, va, perms_attrs); + rem_size -= KB(4); + va += KB(4); + } + + /* Remap complete, re-enable the MMU, unlock the interrupts. */ + + invalidate_tlb_all(); + __set_SCTLR(reg_val); + + arch_irq_unlock(lock_key); +} + +/** + * @brief Maps a 4 kB memory page using a L2 page table entry + * Maps a single 4 kB page of memory from the specified physical + * address to the specified virtual address, using the provided + * attributes and permissions which have already been converted + * from the system's format provided to arch_mem_map() to the + * bits / bit masks used in the L2 page table entry. + * + * @param va 32-bit target virtual address. + * @param pa 32-bit physical address. + * @param perms_attrs Permission and attribute bits in the format + * used in the MMU's L2 page table entries. + */ +static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa, + struct arm_mmu_perms_attrs perms_attrs) +{ + struct arm_mmu_l2_page_table *l2_page_table = NULL; + uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L1_INDEX_MASK; + uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L2_INDEX_MASK; + + /* + * Use the calculated L1 index in order to determine if a L2 page + * table is required in order to complete the current mapping. + * -> See below for an explanation of the possible scenarios. + */ + + if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID || + (l1_page_table.entries[l1_index].undefined.id & ARM_MMU_PTE_ID_SECTION) != 0) { + l2_page_table = arm_mmu_assign_l2_table(pa); + __ASSERT(l2_page_table != NULL, + "Unexpected L2 page table NULL pointer for VA 0x%08X", + va); + } + + /* + * Check what is currently present at the corresponding L1 table entry. + * The following scenarios are possible: + * 1) The L1 PTE's ID bits are zero, as is the rest of the entry. + * In this case, the L1 PTE is currently unused. A new L2 PT to + * refer to in this entry has already been allocated above. + * 2) The L1 PTE's ID bits indicate a L2 PT reference entry (01). + * The corresponding L2 PT's address will be resolved using this + * entry. + * 3) The L1 PTE's ID bits may or may not be zero, and the rest of + * the descriptor contains some non-zero data. This always indicates + * an existing 1 MB section entry in this place. Checking only the + * ID bits wouldn't be enough, as the only way to indicate a section + * with neither R nor W permissions is to set the ID bits to 00 in + * the AP[2:1] permissions model. As we're now about to map a single + * page overlapping with the 1 MB section, the section has to be + * converted into a L2 table. Afterwards, the current page mapping + * can be added/modified. + */ + + if (l1_page_table.entries[l1_index].word == 0) { + /* The matching L1 PT entry is currently unused */ + l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT; + l1_page_table.entries[l1_index].l2_page_table_ref.zero0 = 0; + l1_page_table.entries[l1_index].l2_page_table_ref.zero1 = 0; + l1_page_table.entries[l1_index].l2_page_table_ref.impl_def = 0; + l1_page_table.entries[l1_index].l2_page_table_ref.domain = 0; /* TODO */ + l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = + perms_attrs.non_sec; + l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address = + (((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) & + ARM_MMU_PT_L2_ADDR_MASK); + } else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) { + /* The matching L1 PT entry already points to a L2 PT */ + l2_page_table = (struct arm_mmu_l2_page_table *) + ((l1_page_table.entries[l1_index].word & + (ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT))); + /* + * The only configuration bit contained in the L2 PT entry is the + * NS bit. Set it according to the attributes passed to this function, + * warn if there is a mismatch between the current page's NS attribute + * value and the value currently contained in the L2 PT entry. + */ + if (l1_page_table.entries[l1_index].l2_page_table_ref.non_sec != + perms_attrs.non_sec) { + LOG_WRN("NS bit mismatch in L2 PT reference at L1 index [%u], " + "re-configuring from %u to %u", + l1_index, + l1_page_table.entries[l1_index].l2_page_table_ref.non_sec, + perms_attrs.non_sec); + l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = + perms_attrs.non_sec; + } + } else if (l1_page_table.entries[l1_index].undefined.reserved != 0) { + /* + * The matching L1 PT entry currently holds a 1 MB section entry + * in order to save a L2 table (as it's neither completely blank + * nor a L2 PT reference), but now we have to map an overlapping + * 4 kB page, so the section entry must be converted to a L2 table + * first before the individual L2 entry for the page to be mapped is + * accessed. A blank L2 PT has already been assigned above. + */ + arm_mmu_remap_l1_section_to_l2_table(va, l2_page_table); + } + + /* + * If the matching L2 PTE is blank, increment the number of used entries + * in the L2 table. If the L2 PTE already contains some data, we're re- + * placing the entry's data instead, the used entry count remains unchanged. + * Once again, checking the ID bits might be misleading if the PTE declares + * a page which has neither R nor W permissions. + */ + if (l2_page_table->entries[l2_index].word == 0) { + arm_mmu_inc_l2_table_entries(l2_page_table); + } + + l2_page_table->entries[l2_index].l2_page_4k.id = + (ARM_MMU_PTE_ID_SMALL_PAGE & perms_attrs.id_mask); + l2_page_table->entries[l2_index].l2_page_4k.id |= perms_attrs.exec_never; /* XN in [0] */ + l2_page_table->entries[l2_index].l2_page_4k.bufferable = perms_attrs.bufferable; + l2_page_table->entries[l2_index].l2_page_4k.cacheable = perms_attrs.cacheable; + l2_page_table->entries[l2_index].l2_page_4k.acc_perms10 = + ((perms_attrs.acc_perms & 0x1) << 1) | 0x1; + l2_page_table->entries[l2_index].l2_page_4k.tex = perms_attrs.tex; + l2_page_table->entries[l2_index].l2_page_4k.acc_perms2 = + ((perms_attrs.acc_perms >> 1) & 0x1); + l2_page_table->entries[l2_index].l2_page_4k.shared = perms_attrs.shared; + l2_page_table->entries[l2_index].l2_page_4k.not_global = perms_attrs.not_global; + l2_page_table->entries[l2_index].l2_page_4k.pa_base = + ((pa >> ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT) & + ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_MASK); +} + +/** + * @brief Unmaps a 4 kB memory page by clearing its L2 page table entry + * Unmaps a single 4 kB page of memory from the specified virtual + * address by clearing its respective L2 page table entry. + * + * @param va 32-bit virtual address to be unmapped. + */ +static void arm_mmu_l2_unmap_page(uint32_t va) +{ + struct arm_mmu_l2_page_table *l2_page_table; + uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L1_INDEX_MASK; + uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L2_INDEX_MASK; + + if (l1_page_table.entries[l1_index].undefined.id != ARM_MMU_PTE_ID_L2_PT) { + /* + * No L2 PT currently exists for the given VA - this should be + * tolerated without an error, just as in the case that while + * a L2 PT exists, the corresponding PTE is blank - see explanation + * below, the same applies here. + */ + return; + } + + l2_page_table = (struct arm_mmu_l2_page_table *) + ((l1_page_table.entries[l1_index].word & + (ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT))); + + if (l2_page_table->entries[l2_index].word == 0) { + /* + * We're supposed to unmap a page at the given VA, but there currently + * isn't anything mapped at this address, the L2 PTE is blank. + * -> This is normal if a memory area is being mapped via k_mem_map, + * which contains two calls to arch_mem_unmap (which effectively end up + * here) in order to unmap the leading and trailing guard pages. + * Therefore, it has to be expected that unmap calls are made for unmapped + * memory which hasn't been in use before. + * -> Just return, don't decrement the entry counter of the corresponding + * L2 page table, as we're not actually clearing any PTEs. + */ + return; + } + + if ((l2_page_table->entries[l2_index].undefined.id & ARM_MMU_PTE_ID_SMALL_PAGE) != + ARM_MMU_PTE_ID_SMALL_PAGE) { + LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid " + "page table entry type in level 2 page table at " + "L1 index [%u], L2 index [%u]", va, l1_index, l2_index); + return; + } + + l2_page_table->entries[l2_index].word = 0; + + arm_mmu_dec_l2_table_entries(l2_page_table); +} + +/** + * @brief MMU boot-time initialization function + * Initializes the MMU at boot time. Sets up the page tables and + * applies any specified memory mappings for either the different + * sections of the Zephyr binary image, or for device memory as + * specified at the SoC level. + * + * @retval Always 0, errors are handled by assertions. + */ +int z_arm_mmu_init(void) +{ + uint32_t mem_range; + uint32_t pa; + uint32_t va; + uint32_t attrs; + uint32_t pt_attrs = 0; + uint32_t rem_size; + uint32_t reg_val = 0; + struct arm_mmu_perms_attrs perms_attrs; + + __ASSERT(KB(4) == CONFIG_MMU_PAGE_SIZE, + "MMU_PAGE_SIZE value %u is invalid, only 4 kB pages are supported\n", + CONFIG_MMU_PAGE_SIZE); + + /* Set up the memory regions pre-defined by the image */ + for (mem_range = 0; mem_range < ARRAY_SIZE(mmu_zephyr_ranges); mem_range++) { + pa = mmu_zephyr_ranges[mem_range].start; + rem_size = mmu_zephyr_ranges[mem_range].end - pa; + attrs = mmu_zephyr_ranges[mem_range].attrs; + perms_attrs = arm_mmu_convert_attr_flags(attrs); + + /* + * Check if the L1 page table is within the region currently + * being mapped. If so, store the permissions and attributes + * of the current section. This information is required when + * writing to the TTBR0 register. + */ + if (((uint32_t)&l1_page_table >= pa) && + ((uint32_t)&l1_page_table < (pa + rem_size))) { + pt_attrs = attrs; + } + + while (rem_size > 0) { + if (rem_size >= MB(1) && (pa & 0xFFFFF) == 0 && + (attrs & MATTR_MAY_MAP_L1_SECTION)) { + /* + * Remaining area size > 1 MB & matching alignment + * -> map a 1 MB section instead of individual 4 kB + * pages with identical configuration. + */ + arm_mmu_l1_map_section(pa, pa, perms_attrs); + rem_size -= MB(1); + pa += MB(1); + } else { + arm_mmu_l2_map_page(pa, pa, perms_attrs); + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; + pa += KB(4); + } + } + } + + /* Set up the memory regions defined at the SoC level */ + for (mem_range = 0; mem_range < mmu_config.num_regions; mem_range++) { + pa = (uint32_t)(mmu_config.mmu_regions[mem_range].base_pa); + va = (uint32_t)(mmu_config.mmu_regions[mem_range].base_va); + rem_size = (uint32_t)(mmu_config.mmu_regions[mem_range].size); + attrs = mmu_config.mmu_regions[mem_range].attrs; + perms_attrs = arm_mmu_convert_attr_flags(attrs); + + while (rem_size > 0) { + arm_mmu_l2_map_page(va, pa, perms_attrs); + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; + va += KB(4); + pa += KB(4); + } + } + + /* Clear TTBR1 */ + __asm__ __volatile__("mcr p15, 0, %0, c2, c0, 1" : : "r"(reg_val)); + + /* Write TTBCR: EAE, security not yet relevant, N[2:0] = 0 */ + __asm__ __volatile__("mcr p15, 0, %0, c2, c0, 2" + : : "r"(reg_val)); + + /* Write TTBR0 */ + reg_val = ((uint32_t)&l1_page_table.entries[0] & ~0x3FFF); + + /* + * Set IRGN, RGN, S in TTBR0 based on the configuration of the + * memory area the actual page tables are located in. + */ + if (pt_attrs & MATTR_SHARED) { + reg_val |= ARM_MMU_TTBR_SHAREABLE_BIT; + } + + if (pt_attrs & MATTR_CACHE_OUTER_WB_WA) { + reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_WA_CACHEABLE << + ARM_MMU_TTBR_RGN_SHIFT); + } else if (pt_attrs & MATTR_CACHE_OUTER_WT_nWA) { + reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WT_CACHEABLE << + ARM_MMU_TTBR_RGN_SHIFT); + } else if (pt_attrs & MATTR_CACHE_OUTER_WB_nWA) { + reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_nWA_CACHEABLE << + ARM_MMU_TTBR_RGN_SHIFT); + } + + if (pt_attrs & MATTR_CACHE_INNER_WB_WA) { + reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY; + } else if (pt_attrs & MATTR_CACHE_INNER_WT_nWA) { + reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY; + } else if (pt_attrs & MATTR_CACHE_INNER_WB_nWA) { + reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY; + reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY; + } + + __set_TTBR0(reg_val); + + /* Write DACR -> all domains to client = 01b. */ + reg_val = ARM_MMU_DACR_ALL_DOMAINS_CLIENT; + __set_DACR(reg_val); + + invalidate_tlb_all(); + + /* Enable the MMU and Cache in SCTLR */ + reg_val = __get_SCTLR(); + reg_val |= ARM_MMU_SCTLR_AFE_BIT; + reg_val |= ARM_MMU_SCTLR_ICACHE_ENABLE_BIT; + reg_val |= ARM_MMU_SCTLR_DCACHE_ENABLE_BIT; + reg_val |= ARM_MMU_SCTLR_MMU_ENABLE_BIT; + __set_SCTLR(reg_val); + + return 0; +} + +/** + * @brief ARMv7-specific implementation of memory mapping at run-time + * Maps memory according to the parameters provided by the caller + * at run-time. + * + * @param virt 32-bit target virtual address. + * @param phys 32-bit physical address. + * @param size Size (in bytes) of the memory area to map. + * @param flags Memory attributes & permissions. Comp. K_MEM_... + * flags in sys/mem_manage.h. + * @retval 0 on success, -EINVAL if an invalid parameter is detected. + */ +static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) +{ + uint32_t va = (uint32_t)virt; + uint32_t pa = (uint32_t)phys; + uint32_t rem_size = (uint32_t)size; + uint32_t conv_flags = MPERM_R; + struct arm_mmu_perms_attrs perms_attrs; + int key; + + if (size == 0) { + LOG_ERR("Cannot map physical memory at 0x%08X: invalid " + "zero size", (uint32_t)phys); + return -EINVAL; + } + + switch (flags & K_MEM_CACHE_MASK) { + + case K_MEM_CACHE_NONE: + default: + conv_flags |= MT_DEVICE; + break; + case K_MEM_CACHE_WB: + conv_flags |= MT_NORMAL; + conv_flags |= MATTR_SHARED; + if (flags & K_MEM_PERM_RW) { + conv_flags |= MATTR_CACHE_OUTER_WB_WA; + conv_flags |= MATTR_CACHE_INNER_WB_WA; + } else { + conv_flags |= MATTR_CACHE_OUTER_WB_nWA; + conv_flags |= MATTR_CACHE_INNER_WB_nWA; + } + break; + case K_MEM_CACHE_WT: + conv_flags |= MT_NORMAL; + conv_flags |= MATTR_SHARED; + conv_flags |= MATTR_CACHE_OUTER_WT_nWA; + conv_flags |= MATTR_CACHE_INNER_WT_nWA; + break; + + } + + if (flags & K_MEM_PERM_RW) { + conv_flags |= MPERM_W; + } + if (flags & K_MEM_PERM_EXEC) { + conv_flags |= MPERM_X; + } + + perms_attrs = arm_mmu_convert_attr_flags(conv_flags); + + key = arch_irq_lock(); + + while (rem_size > 0) { + arm_mmu_l2_map_page(va, pa, perms_attrs); + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; + va += KB(4); + pa += KB(4); + } + + arch_irq_unlock(key); + + return 0; +} + +/** + * @brief Arch-specific wrapper function for memory mapping at run-time + * Maps memory according to the parameters provided by the caller + * at run-time. This function wraps the ARMv7 MMU specific implementation + * #__arch_mem_map() for the upper layers of the memory management. + * If the map operation fails, a kernel panic will be triggered. + * + * @param virt 32-bit target virtual address. + * @param phys 32-bit physical address. + * @param size Size (in bytes) of the memory area to map. + * @param flags Memory attributes & permissions. Comp. K_MEM_... + * flags in sys/mem_manage.h. + */ +void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) +{ + int ret = __arch_mem_map(virt, phys, size, flags); + + if (ret) { + LOG_ERR("__arch_mem_map() returned %d", ret); + k_panic(); + } else { + invalidate_tlb_all(); + } +} + +/** + * @brief ARMv7-specific implementation of memory unmapping at run-time + * Unmaps memory according to the parameters provided by the caller + * at run-time. + * + * @param addr 32-bit virtual address to unmap. + * @param size Size (in bytes) of the memory area to unmap. + * @retval 0 on success, -EINVAL if an invalid parameter is detected. + */ +static int __arch_mem_unmap(void *addr, size_t size) +{ + uint32_t va = (uint32_t)addr; + uint32_t rem_size = (uint32_t)size; + int key; + + if (addr == NULL) { + LOG_ERR("Cannot unmap virtual memory: invalid NULL pointer"); + return -EINVAL; + } + + if (size == 0) { + LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid " + "zero size", (uint32_t)addr); + return -EINVAL; + } + + key = arch_irq_lock(); + + while (rem_size > 0) { + arm_mmu_l2_unmap_page(va); + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; + va += KB(4); + } + + arch_irq_unlock(key); + + return 0; +} + +/** + * @brief Arch-specific wrapper function for memory unmapping at run-time + * Unmaps memory according to the parameters provided by the caller + * at run-time. This function wraps the ARMv7 MMU specific implementation + * #__arch_mem_unmap() for the upper layers of the memory management. + * + * @param addr 32-bit virtual address to unmap. + * @param size Size (in bytes) of the memory area to unmap. + */ +void arch_mem_unmap(void *addr, size_t size) +{ + int ret = __arch_mem_unmap(addr, size); + + if (ret) { + LOG_ERR("__arch_mem_unmap() returned %d", ret); + } else { + invalidate_tlb_all(); + } +} + +/** + * @brief Arch-specific virtual-to-physical address resolver function + * ARMv7 MMU specific implementation of a function that resolves the + * physical address corresponding to the given virtual address. + * + * @param virt 32-bit target virtual address to resolve. + * @param phys Pointer to a variable to which the resolved physical + * address will be written. May be NULL if this information + * is not actually required by the caller. + * @retval 0 if the physical address corresponding to the specified + * virtual address could be resolved successfully, -EFAULT + * if the specified virtual address is not currently mapped. + */ +int arch_page_phys_get(void *virt, uintptr_t *phys) +{ + uint32_t l1_index = ((uint32_t)virt >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L1_INDEX_MASK; + uint32_t l2_index = ((uint32_t)virt >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) & + ARM_MMU_PTE_L2_INDEX_MASK; + struct arm_mmu_l2_page_table *l2_page_table; + + uint32_t pa_resolved = 0; + uint32_t l2_pt_resolved; + + int rc = 0; + int key; + + key = arch_irq_lock(); + + if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_SECTION) { + /* + * If the virtual address points to a level 1 PTE whose ID bits + * identify it as a 1 MB section entry rather than a level 2 PT + * entry, the given VA belongs to a memory region used by the + * Zephyr image itself - it is only for those static regions that + * L1 Section entries are used to save L2 tables if a sufficient- + * ly large block of memory is specified. The memory regions be- + * longing to the Zephyr image are identity mapped -> just return + * the value of the VA as the value of the PA. + */ + pa_resolved = (uint32_t)virt; + } else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) { + /* + * The VA points to a level 1 PTE which re-directs to a level 2 + * PT. -> Assemble the level 2 PT pointer and resolve the PA for + * the specified VA from there. + */ + l2_pt_resolved = + l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address; + l2_pt_resolved <<= ARM_MMU_PT_L2_ADDR_SHIFT; + l2_page_table = (struct arm_mmu_l2_page_table *)l2_pt_resolved; + + /* + * Check if the PTE for the specified VA is actually in use before + * assembling & returning the corresponding PA. k_mem_unmap will + * call this function for the leading & trailing guard pages when + * unmapping a VA. As those guard pages were explicitly unmapped + * when the VA was originally mapped, their L2 PTEs will be empty. + * In that case, the return code of this function must not be 0. + */ + if (l2_page_table->entries[l2_index].word == 0) { + rc = -EFAULT; + } + + pa_resolved = l2_page_table->entries[l2_index].l2_page_4k.pa_base; + pa_resolved <<= ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT; + pa_resolved |= ((uint32_t)virt & ARM_MMU_ADDR_BELOW_PAGE_GRAN_MASK); + } else { + /* The level 1 PTE is invalid -> the specified VA is not mapped */ + rc = -EFAULT; + } + + arch_irq_unlock(key); + + if (phys) { + *phys = (uintptr_t)pa_resolved; + } + return rc; +} + +/* EOF */ diff --git a/arch/arm/core/aarch32/mmu/arm_mmu_priv.h b/arch/arm/core/aarch32/mmu/arm_mmu_priv.h new file mode 100644 index 00000000000..9c69836e18f --- /dev/null +++ b/arch/arm/core/aarch32/mmu/arm_mmu_priv.h @@ -0,0 +1,205 @@ +/* + * ARMv7 MMU support + * + * Private data declarations + * + * Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_ARCH_AARCH32_ARM_MMU_PRIV_H_ +#define ZEPHYR_ARCH_AARCH32_ARM_MMU_PRIV_H_ + +/* + * Comp.: + * ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition + * ARM document ID DDI0406C Rev. d, March 2018 + * L1 / L2 page table entry formats and entry type IDs: + * Chapter B3.5.1, fig. B3-4 and B3-5, p. B3-1323 f. + */ + +#define ARM_MMU_PT_L1_NUM_ENTRIES 4096 +#define ARM_MMU_PT_L2_NUM_ENTRIES 256 + +#define ARM_MMU_PTE_L1_INDEX_PA_SHIFT 20 +#define ARM_MMU_PTE_L1_INDEX_MASK 0xFFF +#define ARM_MMU_PTE_L2_INDEX_PA_SHIFT 12 +#define ARM_MMU_PTE_L2_INDEX_MASK 0xFF +#define ARM_MMU_PT_L2_ADDR_SHIFT 10 +#define ARM_MMU_PT_L2_ADDR_MASK 0x3FFFFF +#define ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT 12 +#define ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_MASK 0xFFFFF +#define ARM_MMU_ADDR_BELOW_PAGE_GRAN_MASK 0xFFF + +#define ARM_MMU_PTE_ID_INVALID 0x0 +#define ARM_MMU_PTE_ID_L2_PT 0x1 +#define ARM_MMU_PTE_ID_SECTION 0x2 +#define ARM_MMU_PTE_ID_LARGE_PAGE 0x1 +#define ARM_MMU_PTE_ID_SMALL_PAGE 0x2 + +#define ARM_MMU_PERMS_AP2_DISABLE_WR 0x2 +#define ARM_MMU_PERMS_AP1_ENABLE_PL0 0x1 +#define ARM_MMU_TEX2_CACHEABLE_MEMORY 0x4 + +#define ARM_MMU_TEX_CACHE_ATTRS_WB_WA 0x1 +#define ARM_MMU_TEX_CACHE_ATTRS_WT_nWA 0x2 +#define ARM_MMU_TEX_CACHE_ATTRS_WB_nWA 0x3 +#define ARM_MMU_C_CACHE_ATTRS_WB_WA 0 +#define ARM_MMU_B_CACHE_ATTRS_WB_WA 1 +#define ARM_MMU_C_CACHE_ATTRS_WT_nWA 1 +#define ARM_MMU_B_CACHE_ATTRS_WT_nWA 0 +#define ARM_MMU_C_CACHE_ATTRS_WB_nWA 1 +#define ARM_MMU_B_CACHE_ATTRS_WB_nWA 1 + +/* + * The following defines might vary if support for CPUs without + * the multiprocessor extensions was to be implemented: + */ + +#define ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY BIT(6) +#define ARM_MMU_TTBR_NOS_BIT BIT(5) +#define ARM_MMU_TTBR_RGN_OUTER_NON_CACHEABLE 0x0 +#define ARM_MMU_TTBR_RGN_OUTER_WB_WA_CACHEABLE 0x1 +#define ARM_MMU_TTBR_RGN_OUTER_WT_CACHEABLE 0x2 +#define ARM_MMU_TTBR_RGN_OUTER_WB_nWA_CACHEABLE 0x3 +#define ARM_MMU_TTBR_RGN_SHIFT 3 +#define ARM_MMU_TTBR_SHAREABLE_BIT BIT(1) +#define ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY BIT(0) +#define ARM_MMU_TTBR_CACHEABLE_BIT_NON_MP_ONLY BIT(0) + +/* <-- end MP-/non-MP-specific */ + +#define ARM_MMU_DOMAIN_OS 0 +#define ARM_MMU_DOMAIN_DEVICE 1 +#define ARM_MMU_DACR_ALL_DOMAINS_CLIENT 0x55555555 + +#define ARM_MMU_SCTLR_AFE_BIT BIT(29) +#define ARM_MMU_SCTLR_TEX_REMAP_ENABLE_BIT BIT(28) +#define ARM_MMU_SCTLR_HA_BIT BIT(17) +#define ARM_MMU_SCTLR_ICACHE_ENABLE_BIT BIT(12) +#define ARM_MMU_SCTLR_DCACHE_ENABLE_BIT BIT(2) +#define ARM_MMU_SCTLR_CHK_ALIGN_ENABLE_BIT BIT(1) +#define ARM_MMU_SCTLR_MMU_ENABLE_BIT BIT(0) + +#define ARM_MMU_L2_PT_INDEX(pt) ((uint32_t)pt - (uint32_t)l2_page_tables) /\ + sizeof(struct arm_mmu_l2_page_table); + +union arm_mmu_l1_page_table_entry { + struct { + uint32_t id : 2; /* [00] */ + uint32_t bufferable : 1; + uint32_t cacheable : 1; + uint32_t exec_never : 1; + uint32_t domain : 4; + uint32_t impl_def : 1; + uint32_t acc_perms10 : 2; + uint32_t tex : 3; + uint32_t acc_perms2 : 1; + uint32_t shared : 1; + uint32_t not_global : 1; + uint32_t zero : 1; + uint32_t non_sec : 1; + uint32_t base_address : 12; /* [31] */ + } l1_section_1m; + struct { + uint32_t id : 2; /* [00] */ + uint32_t zero0 : 1; /* PXN if avail. */ + uint32_t non_sec : 1; + uint32_t zero1 : 1; + uint32_t domain : 4; + uint32_t impl_def : 1; + uint32_t l2_page_table_address : 22; /* [31] */ + } l2_page_table_ref; + struct { + uint32_t id : 2; /* [00] */ + uint32_t reserved : 30; /* [31] */ + } undefined; + uint32_t word; +}; + +struct arm_mmu_l1_page_table { + union arm_mmu_l1_page_table_entry entries[ARM_MMU_PT_L1_NUM_ENTRIES]; +}; + +union arm_mmu_l2_page_table_entry { + struct { + uint32_t id : 2; /* [00] */ + uint32_t bufferable : 1; + uint32_t cacheable : 1; + uint32_t acc_perms10 : 2; + uint32_t tex : 3; + uint32_t acc_perms2 : 1; + uint32_t shared : 1; + uint32_t not_global : 1; + uint32_t pa_base : 20; /* [31] */ + } l2_page_4k; + struct { + uint32_t id : 2; /* [00] */ + uint32_t bufferable : 1; + uint32_t cacheable : 1; + uint32_t acc_perms10 : 2; + uint32_t zero : 3; + uint32_t acc_perms2 : 1; + uint32_t shared : 1; + uint32_t not_global : 1; + uint32_t tex : 3; + uint32_t exec_never : 1; + uint32_t pa_base : 16; /* [31] */ + } l2_page_64k; + struct { + uint32_t id : 2; /* [00] */ + uint32_t reserved : 30; /* [31] */ + } undefined; + uint32_t word; +}; + +struct arm_mmu_l2_page_table { + union arm_mmu_l2_page_table_entry entries[ARM_MMU_PT_L2_NUM_ENTRIES]; +}; + +/* + * Data structure for L2 table usage tracking, contains a + * L1 index reference if the respective L2 table is in use. + */ + +struct arm_mmu_l2_page_table_status { + uint32_t l1_index : 12; + uint32_t entries : 9; + uint32_t reserved : 11; +}; + +/* + * Data structure used to describe memory areas defined by the + * current Zephyr image, for which an identity mapping (pa = va) + * will be set up. Those memory areas are processed during the + * MMU initialization. + */ +struct arm_mmu_flat_range { + const char *name; + uint32_t start; + uint32_t end; + uint32_t attrs; +}; + +/* + * Data structure containing the memory attributes and permissions + * data derived from a memory region's attr flags word in the format + * required for setting up the corresponding PTEs. + */ +struct arm_mmu_perms_attrs { + uint32_t acc_perms : 2; + uint32_t bufferable : 1; + uint32_t cacheable : 1; + uint32_t not_global : 1; + uint32_t non_sec : 1; + uint32_t shared : 1; + uint32_t tex : 3; + uint32_t exec_never : 1; + uint32_t id_mask : 2; + uint32_t domain : 4; + uint32_t reserved : 15; +}; + +#endif /* ZEPHYR_ARCH_AARCH32_ARM_MMU_PRIV_H_ */ + +/* EOF */ diff --git a/arch/arm/include/aarch32/kernel_arch_func.h b/arch/arm/include/aarch32/kernel_arch_func.h index 73eb0246584..08f4a6d92b5 100644 --- a/arch/arm/include/aarch32/kernel_arch_func.h +++ b/arch/arm/include/aarch32/kernel_arch_func.h @@ -35,6 +35,9 @@ extern void z_arm_configure_static_mpu_regions(void); extern void z_arm_configure_dynamic_mpu_regions(struct k_thread *thread); extern int z_arm_mpu_init(void); #endif /* CONFIG_ARM_MPU */ +#ifdef CONFIG_ARM_AARCH32_MMU +extern int z_arm_mmu_init(void); +#endif /* CONFIG_ARM_AARCH32_MMU */ static ALWAYS_INLINE void arch_kernel_init(void) { @@ -52,7 +55,10 @@ static ALWAYS_INLINE void arch_kernel_init(void) * This function is invoked once, upon system initialization. */ z_arm_configure_static_mpu_regions(); -#endif +#endif /* CONFIG_ARM_MPU */ +#if defined(CONFIG_ARM_AARCH32_MMU) + z_arm_mmu_init(); +#endif /* CONFIG_ARM_AARCH32_MMU */ } static ALWAYS_INLINE void diff --git a/include/arch/arm/aarch32/arch.h b/include/arch/arm/aarch32/arch.h index 8e1b210d12f..0c203f80dae 100644 --- a/include/arch/arm/aarch32/arch.h +++ b/include/arch/arm/aarch32/arch.h @@ -71,6 +71,8 @@ extern "C" { */ #if defined(CONFIG_USERSPACE) #define Z_THREAD_MIN_STACK_ALIGN CONFIG_ARM_MPU_REGION_MIN_ALIGN_AND_SIZE +#elif defined(CONFIG_ARM_MMU) +#define Z_THREAD_MIN_STACK_ALIGN CONFIG_ARM_MMU_REGION_MIN_ALIGN_AND_SIZE #else #define Z_THREAD_MIN_STACK_ALIGN ARCH_STACK_PTR_ALIGN #endif @@ -190,6 +192,9 @@ extern "C" { #include #endif /* CONFIG_CPU_HAS_NXP_MPU */ #endif /* CONFIG_ARM_MPU */ +#ifdef CONFIG_ARM_AARCH32_MMU +#include +#endif /* CONFIG_ARM_AARCH32_MMU */ #ifdef __cplusplus } diff --git a/include/arch/arm/aarch32/cortex_a_r/scripts/linker.ld b/include/arch/arm/aarch32/cortex_a_r/scripts/linker.ld index 57a0dd33b78..49ee3467b8a 100644 --- a/include/arch/arm/aarch32/cortex_a_r/scripts/linker.ld +++ b/include/arch/arm/aarch32/cortex_a_r/scripts/linker.ld @@ -58,6 +58,8 @@ */ #if defined(CONFIG_ARM_MPU_REGION_MIN_ALIGN_AND_SIZE) _region_min_align = CONFIG_ARM_MPU_REGION_MIN_ALIGN_AND_SIZE; +#elif defined(CONFIG_ARM_AARCH32_MMU) +_region_min_align = CONFIG_MMU_PAGE_SIZE; #else /* If building without MPU support, use default 4-byte alignment. */ _region_min_align = 4; @@ -72,6 +74,8 @@ _region_min_align = 4; . = ALIGN(_region_min_align) #endif +#define BSS_ALIGN ALIGN(_region_min_align) + MEMORY { FLASH (rx) : ORIGIN = ROM_ADDR, LENGTH = ROM_SIZE @@ -127,7 +131,11 @@ SECTIONS SECTION_PROLOGUE(_TEXT_SECTION_NAME,,) { + . = ALIGN(_region_min_align); __text_region_start = .; +#ifndef CONFIG_XIP + z_mapped_start = .; +#endif #include @@ -144,6 +152,7 @@ SECTIONS } GROUP_LINK_IN(ROMABLE_REGION) __text_region_end = .; + . = ALIGN(_region_min_align); #if defined (CONFIG_CPLUSPLUS) SECTION_PROLOGUE(.ARM.extab,,) @@ -170,6 +179,7 @@ SECTIONS __exidx_end = .; } GROUP_LINK_IN(ROMABLE_REGION) + . = ALIGN(_region_min_align); __rodata_region_start = .; #include @@ -210,6 +220,7 @@ SECTIONS __rodata_region_end = .; __rom_region_end = .; + MPU_ALIGN(__rodata_region_end - __rom_region_start); _image_rom_end_order = (LOG2CEIL(__rom_region_end) - 1) << 1; GROUP_END(ROMABLE_REGION) @@ -234,6 +245,9 @@ SECTIONS */ . = ALIGN(_region_min_align); _image_ram_start = .; +#ifdef CONFIG_XIP + z_mapped_start = .; +#endif /* Located in generated directory. This file is populated by the * zephyr_linker_sources() Cmake function. @@ -250,7 +264,7 @@ SECTIONS _app_smem_rom_start = LOADADDR(_APP_SMEM_SECTION_NAME); #endif /* CONFIG_USERSPACE */ - SECTION_DATA_PROLOGUE(_BSS_SECTION_NAME,(NOLOAD),) + SECTION_DATA_PROLOGUE(_BSS_SECTION_NAME,(NOLOAD), BSS_ALIGN) { /* * For performance, BSS section is assumed to be 4 byte aligned and @@ -316,8 +330,10 @@ SECTIONS /* Define linker symbols */ + . = ALIGN(_region_min_align); _image_ram_end = .; _end = .; /* end of image */ + z_mapped_end = .; __kernel_ram_end = RAM_ADDR + RAM_SIZE; __kernel_ram_size = __kernel_ram_end - __kernel_ram_start; diff --git a/include/arch/arm/aarch32/mmu/arm_mmu.h b/include/arch/arm/aarch32/mmu/arm_mmu.h new file mode 100644 index 00000000000..a4f0fe34b4d --- /dev/null +++ b/include/arch/arm/aarch32/mmu/arm_mmu.h @@ -0,0 +1,105 @@ +/* + * ARMv7 MMU support + * + * Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_ARCH_AARCH32_ARM_MMU_H_ +#define ZEPHYR_INCLUDE_ARCH_AARCH32_ARM_MMU_H_ + +#ifndef _ASMLANGUAGE + +/* + * Comp.: + * ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition, + * ARM document ID DDI0406C Rev. d, March 2018 + * Memory type definitions: + * Table B3-10, chap. B3.8.2, p. B3-1363f. + * Outer / inner cache attributes for cacheable memory: + * Table B3-11, chap. B3.8.2, p. B3-1364 + */ + +/* + * The following definitions are used when specifying a memory + * range to be mapped at boot time using the MMU_REGION_ENTRY + * macro. + */ +#define MT_STRONGLY_ORDERED BIT(0) +#define MT_DEVICE BIT(1) +#define MT_NORMAL BIT(2) +#define MT_MASK 0x7 + +#define MPERM_R BIT(3) +#define MPERM_W BIT(4) +#define MPERM_X BIT(5) +#define MPERM_UNPRIVILEGED BIT(6) + +#define MATTR_NON_SECURE BIT(7) +#define MATTR_NON_GLOBAL BIT(8) +#define MATTR_SHARED BIT(9) +#define MATTR_CACHE_OUTER_WB_WA BIT(10) +#define MATTR_CACHE_OUTER_WT_nWA BIT(11) +#define MATTR_CACHE_OUTER_WB_nWA BIT(12) +#define MATTR_CACHE_INNER_WB_WA BIT(13) +#define MATTR_CACHE_INNER_WT_nWA BIT(14) +#define MATTR_CACHE_INNER_WB_nWA BIT(15) + +#define MATTR_MAY_MAP_L1_SECTION BIT(16) + +/* + * The following macros are used for adding constant entries + * mmu_regions array of the mmu_config struct. Use MMU_REGION_ENTRY + * for the specification of mappings whose PA and VA differ, + * the use of MMU_REGION_FLAT_ENTRY always results in an identity + * mapping, which are used for the mappings of the Zephyr image's + * code and data. + */ +#define MMU_REGION_ENTRY(_name, _base_pa, _base_va, _size, _attrs) \ + {\ + .name = _name, \ + .base_pa = _base_pa, \ + .base_va = _base_va, \ + .size = _size, \ + .attrs = _attrs, \ + } + +#define MMU_REGION_FLAT_ENTRY(name, adr, sz, attrs) \ + MMU_REGION_ENTRY(name, adr, adr, sz, attrs) + +/* Region definition data structure */ +struct arm_mmu_region { + /* Region Base Physical Address */ + uintptr_t base_pa; + /* Region Base Virtual Address */ + uintptr_t base_va; + /* Region size */ + size_t size; + /* Region Name */ + const char *name; + /* Region Attributes */ + uint32_t attrs; +}; + +/* MMU configuration data structure */ +struct arm_mmu_config { + /* Number of regions */ + uint32_t num_regions; + /* Regions */ + const struct arm_mmu_region *mmu_regions; +}; + +/* + * Reference to the MMU configuration. + * + * This struct is defined and populated for each SoC (in the SoC definition), + * and holds the build-time configuration information for the fixed MMU + * regions enabled during kernel initialization. + */ +extern const struct arm_mmu_config mmu_config; + +int z_arm_mmu_init(void); + +#endif /* _ASMLANGUAGE */ + +#endif /* ZEPHYR_INCLUDE_ARCH_AARCH32_ARM_MMU_H_ */