/* * ARMv7 MMU support * * This implementation supports the Short-descriptor translation * table format. The standard page size is 4 kB, 1 MB sections * are only used for mapping the code and data of the Zephyr image. * Secure mode and PL1 is always assumed. LPAE and PXN extensions * as well as TEX remapping are not supported. The AP[2:1] plus * Access flag permissions model is used, as the AP[2:0] model is * deprecated. As the AP[2:1] model can only disable write access, * the read permission flag is always implied. * * Reference documentation: * ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition, * ARM document ID DDI0406C Rev. d, March 2018 * * Copyright (c) 2021 Weidmueller Interface GmbH & Co. KG * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include #include #include #include #include #include #include #include "arm_mmu_priv.h" LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); /* Level 1 page table: always required, must be 16k-aligned */ static struct arm_mmu_l1_page_table l1_page_table __aligned(KB(16)) = {0}; /* * Array of level 2 page tables with 4k granularity: * each table covers a range of 1 MB, the number of L2 tables * is configurable. */ static struct arm_mmu_l2_page_table l2_page_tables[CONFIG_ARM_MMU_NUM_L2_TABLES] __aligned(KB(1)) = {0}; /* * For each level 2 page table, a separate dataset tracks * if the respective table is in use, if so, to which 1 MB * virtual address range it is assigned, and how many entries, * each mapping a 4 kB page, it currently contains. */ static struct arm_mmu_l2_page_table_status l2_page_tables_status[CONFIG_ARM_MMU_NUM_L2_TABLES] = {0}; /* Available L2 tables count & next free index for an L2 table request */ static uint32_t arm_mmu_l2_tables_free = CONFIG_ARM_MMU_NUM_L2_TABLES; static uint32_t arm_mmu_l2_next_free_table; /* * Static definition of all code & data memory regions of the * current Zephyr image. This information must be available & * processed upon MMU initialization. */ static const struct arm_mmu_flat_range mmu_zephyr_ranges[] = { /* * Mark the zephyr execution regions (data, bss, noinit, etc.) * cacheable, read / write and non-executable */ { .name = "zephyr_data", .start = (uint32_t)_image_ram_start, .end = (uint32_t)_image_ram_end, .attrs = MT_NORMAL | MATTR_SHARED | MPERM_R | MPERM_W | MATTR_CACHE_OUTER_WB_WA | MATTR_CACHE_INNER_WB_WA}, /* Mark text segment cacheable, read only and executable */ { .name = "zephyr_code", .start = (uint32_t)__text_region_start, .end = (uint32_t)__text_region_end, .attrs = MT_NORMAL | MATTR_SHARED | /* The code needs to have write permission in order for * software breakpoints (which modify instructions) to work */ #if defined(CONFIG_GDBSTUB) MPERM_R | MPERM_X | MPERM_W | #else MPERM_R | MPERM_X | #endif MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA | MATTR_MAY_MAP_L1_SECTION}, /* Mark rodata segment cacheable, read only and non-executable */ { .name = "zephyr_rodata", .start = (uint32_t)__rodata_region_start, .end = (uint32_t)__rodata_region_end, .attrs = MT_NORMAL | MATTR_SHARED | MPERM_R | MATTR_CACHE_OUTER_WB_nWA | MATTR_CACHE_INNER_WB_nWA | MATTR_MAY_MAP_L1_SECTION}, #ifdef CONFIG_NOCACHE_MEMORY /* Mark nocache segment read / write and non-executable */ { .name = "nocache", .start = (uint32_t)_nocache_ram_start, .end = (uint32_t)_nocache_ram_end, .attrs = MT_STRONGLY_ORDERED | MPERM_R | MPERM_W}, #endif }; static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa, struct arm_mmu_perms_attrs perms_attrs); /** * @brief Invalidates the TLB * Helper function which invalidates the entire TLB. This action * is performed whenever the MMU is (re-)enabled or changes to the * page tables are made at run-time, as the TLB might contain entries * which are no longer valid once the changes are applied. */ static void invalidate_tlb_all(void) { __set_TLBIALL(0); /* 0 = opc2 = invalidate entire TLB */ barrier_dsync_fence_full(); barrier_isync_fence_full(); } /** * @brief Returns a free level 2 page table * Initializes and returns the next free L2 page table whenever * a page is to be mapped in a 1 MB virtual address range that * is not yet covered by a level 2 page table. * * @param va 32-bit virtual address to be mapped. * @retval pointer to the L2 table now assigned to the 1 MB * address range the target virtual address is in. */ static struct arm_mmu_l2_page_table *arm_mmu_assign_l2_table(uint32_t va) { struct arm_mmu_l2_page_table *l2_page_table; __ASSERT(arm_mmu_l2_tables_free > 0, "Cannot set up L2 page table for VA 0x%08X: " "no more free L2 page tables available\n", va); __ASSERT(l2_page_tables_status[arm_mmu_l2_next_free_table].entries == 0, "Cannot set up L2 page table for VA 0x%08X: " "expected empty L2 table at index [%u], but the " "entries value is %u\n", va, arm_mmu_l2_next_free_table, l2_page_tables_status[arm_mmu_l2_next_free_table].entries); /* * Store in the status dataset of the L2 table to be returned * which 1 MB virtual address range it is being assigned to. * Set the current page table entry count to 0. */ l2_page_tables_status[arm_mmu_l2_next_free_table].l1_index = ((va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK); l2_page_tables_status[arm_mmu_l2_next_free_table].entries = 0; l2_page_table = &l2_page_tables[arm_mmu_l2_next_free_table]; /* * Decrement the available L2 page table count. As long as at * least one more L2 table is available afterwards, update the * L2 next free table index. If we're about to return the last * available L2 table, calculating a next free table index is * impossible. */ --arm_mmu_l2_tables_free; if (arm_mmu_l2_tables_free > 0) { do { arm_mmu_l2_next_free_table = (arm_mmu_l2_next_free_table + 1) % CONFIG_ARM_MMU_NUM_L2_TABLES; } while (l2_page_tables_status[arm_mmu_l2_next_free_table].entries != 0); } return l2_page_table; } /** * @brief Releases a level 2 page table * Releases a level 2 page table, marking it as no longer in use. * From that point on, it can be re-used for mappings in another * 1 MB virtual address range. This function is called whenever * it is determined during an unmap call at run-time that the page * table entry count in the respective page table has reached 0. * * @param l2_page_table Pointer to L2 page table to be released. */ static void arm_mmu_release_l2_table(struct arm_mmu_l2_page_table *l2_page_table) { uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table); l2_page_tables_status[l2_page_table_index].l1_index = 0; if (arm_mmu_l2_tables_free == 0) { arm_mmu_l2_next_free_table = l2_page_table_index; } ++arm_mmu_l2_tables_free; } /** * @brief Increments the page table entry counter of a L2 page table * Increments the page table entry counter of a level 2 page table. * Contains a check to ensure that no attempts are made to set up * more page table entries than the table can hold. * * @param l2_page_table Pointer to the L2 page table whose entry * counter shall be incremented. */ static void arm_mmu_inc_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table) { uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table); __ASSERT(l2_page_tables_status[l2_page_table_index].entries < ARM_MMU_PT_L2_NUM_ENTRIES, "Cannot increment entry count of the L2 page table at index " "[%u] / addr %p / ref L1[%u]: maximum entry count already reached", l2_page_table_index, l2_page_table, l2_page_tables_status[l2_page_table_index].l1_index); ++l2_page_tables_status[l2_page_table_index].entries; } /** * @brief Decrements the page table entry counter of a L2 page table * Decrements the page table entry counter of a level 2 page table. * Contains a check to ensure that no attempts are made to remove * entries from the respective table that aren't actually there. * * @param l2_page_table Pointer to the L2 page table whose entry * counter shall be decremented. */ static void arm_mmu_dec_l2_table_entries(struct arm_mmu_l2_page_table *l2_page_table) { uint32_t l2_page_table_index = ARM_MMU_L2_PT_INDEX(l2_page_table); __ASSERT(l2_page_tables_status[l2_page_table_index].entries > 0, "Cannot decrement entry count of the L2 page table at index " "[%u] / addr %p / ref L1[%u]: entry count is already zero", l2_page_table_index, l2_page_table, l2_page_tables_status[l2_page_table_index].l1_index); if (--l2_page_tables_status[l2_page_table_index].entries == 0) { arm_mmu_release_l2_table(l2_page_table); } } /** * @brief Converts memory attributes and permissions to MMU format * Converts memory attributes and permissions as used in the boot- * time memory mapping configuration data array (MT_..., MATTR_..., * MPERM_...) to the equivalent bit (field) values used in the MMU's * L1 and L2 page table entries. Contains plausibility checks. * * @param attrs type/attribute/permissions flags word obtained from * an entry of the mmu_config mapping data array. * @retval A struct containing the information from the input flags * word converted to the bits / bit fields used in L1 and * L2 page table entries. */ static struct arm_mmu_perms_attrs arm_mmu_convert_attr_flags(uint32_t attrs) { struct arm_mmu_perms_attrs perms_attrs = {0}; __ASSERT(((attrs & MT_MASK) > 0), "Cannot convert attrs word to PTE control bits: no " "memory type specified"); __ASSERT(!((attrs & MPERM_W) && !(attrs & MPERM_R)), "attrs must not define write permission without read " "permission"); __ASSERT(!((attrs & MPERM_W) && (attrs & MPERM_X)), "attrs must not define executable memory with write " "permission"); /* * The translation of the memory type / permissions / attributes * flags in the attrs word to the TEX, C, B, S and AP bits of the * target PTE is based on the reference manual: * TEX, C, B, S: Table B3-10, chap. B3.8.2, p. B3-1363f. * AP : Table B3-6, chap. B3.7.1, p. B3-1353. * Device / strongly ordered memory is always assigned to a domain * other than that used for normal memory. Assuming that userspace * support utilizing the MMU is eventually implemented, a single * modification of the DACR register when entering/leaving unprivi- * leged mode could be used in order to enable/disable all device * memory access without having to modify any PTs/PTEs. */ if (attrs & MT_STRONGLY_ORDERED) { /* Strongly ordered is always shareable, S bit is ignored */ perms_attrs.tex = 0; perms_attrs.cacheable = 0; perms_attrs.bufferable = 0; perms_attrs.shared = 0; perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE; } else if (attrs & MT_DEVICE) { /* * Shareability of device memory is determined by TEX, C, B. * The S bit is ignored. C is always 0 for device memory. */ perms_attrs.shared = 0; perms_attrs.cacheable = 0; perms_attrs.domain = ARM_MMU_DOMAIN_DEVICE; if (attrs & MATTR_SHARED) { perms_attrs.tex = 0; perms_attrs.bufferable = 1; } else { perms_attrs.tex = 2; perms_attrs.bufferable = 0; } } else if (attrs & MT_NORMAL) { /* * TEX[2] is always 1. TEX[1:0] contain the outer cache attri- * butes encoding, C and B contain the inner cache attributes * encoding. */ perms_attrs.tex |= ARM_MMU_TEX2_CACHEABLE_MEMORY; perms_attrs.domain = ARM_MMU_DOMAIN_OS; /* For normal memory, shareability depends on the S bit */ if (attrs & MATTR_SHARED) { perms_attrs.shared = 1; } if (attrs & MATTR_CACHE_OUTER_WB_WA) { perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_WA; } else if (attrs & MATTR_CACHE_OUTER_WT_nWA) { perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WT_nWA; } else if (attrs & MATTR_CACHE_OUTER_WB_nWA) { perms_attrs.tex |= ARM_MMU_TEX_CACHE_ATTRS_WB_nWA; } if (attrs & MATTR_CACHE_INNER_WB_WA) { perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_WA; perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_WA; } else if (attrs & MATTR_CACHE_INNER_WT_nWA) { perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WT_nWA; perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WT_nWA; } else if (attrs & MATTR_CACHE_INNER_WB_nWA) { perms_attrs.cacheable = ARM_MMU_C_CACHE_ATTRS_WB_nWA; perms_attrs.bufferable = ARM_MMU_B_CACHE_ATTRS_WB_nWA; } } if (attrs & MATTR_NON_SECURE) { perms_attrs.non_sec = 1; } if (attrs & MATTR_NON_GLOBAL) { perms_attrs.not_global = 1; } /* * Up next is the consideration of the case that a PTE shall be configured * for a page that shall not be accessible at all (e.g. guard pages), and * therefore has neither read nor write permissions. In the AP[2:1] access * permission specification model, the only way to indicate this is to * actually mask out the PTE's identifier bits, as otherwise, read permission * is always granted for any valid PTE, it can't be revoked explicitly, * unlike the write permission. */ if (!((attrs & MPERM_R) || (attrs & MPERM_W))) { perms_attrs.id_mask = 0x0; } else { perms_attrs.id_mask = 0x3; } if (!(attrs & MPERM_W)) { perms_attrs.acc_perms |= ARM_MMU_PERMS_AP2_DISABLE_WR; } if (attrs & MPERM_UNPRIVILEGED) { perms_attrs.acc_perms |= ARM_MMU_PERMS_AP1_ENABLE_PL0; } if (!(attrs & MPERM_X)) { perms_attrs.exec_never = 1; } return perms_attrs; } /** * @brief Maps a 1 MB memory range via a level 1 page table entry * Maps a 1 MB memory range using a level 1 page table entry of type * 'section'. This type of entry saves a level 2 page table, but has * two pre-conditions: the memory area to be mapped must contain at * least 1 MB of contiguous memory, starting at an address with suit- * able alignment. This mapping method should only be used for map- * pings for which it is unlikely that the attributes of those mappings * will mappings will change at run-time (e.g. code sections will al- * ways be read-only and executable). Should the case occur that the * permissions or attributes of a subset of a 1 MB section entry shall * be re-configured at run-time, a L1 section entry will be broken * down into 4k segments using a L2 table with identical attributes * before any modifications are performed for the subset of the affec- * ted 1 MB range. This comes with an undeterministic performance * penalty at the time of re-configuration, therefore, any mappings * for which L1 section entries are a valid option, shall be marked in * their declaration with the MATTR_MAY_MAP_L1_SECTION flag. * * @param va 32-bit target virtual address to be mapped. * @param pa 32-bit physical address to be mapped. * @param perms_attrs Permission and attribute bits in the format * used in the MMU's L1 page table entries. */ static void arm_mmu_l1_map_section(uint32_t va, uint32_t pa, struct arm_mmu_perms_attrs perms_attrs) { uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK; __ASSERT(l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID, "Unexpected non-zero L1 PTE ID %u for VA 0x%08X / PA 0x%08X", l1_page_table.entries[l1_index].undefined.id, va, pa); l1_page_table.entries[l1_index].l1_section_1m.id = (ARM_MMU_PTE_ID_SECTION & perms_attrs.id_mask); l1_page_table.entries[l1_index].l1_section_1m.bufferable = perms_attrs.bufferable; l1_page_table.entries[l1_index].l1_section_1m.cacheable = perms_attrs.cacheable; l1_page_table.entries[l1_index].l1_section_1m.exec_never = perms_attrs.exec_never; l1_page_table.entries[l1_index].l1_section_1m.domain = perms_attrs.domain; l1_page_table.entries[l1_index].l1_section_1m.impl_def = 0; l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 = ((perms_attrs.acc_perms & 0x1) << 1) | 0x1; l1_page_table.entries[l1_index].l1_section_1m.tex = perms_attrs.tex; l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 = (perms_attrs.acc_perms >> 1) & 0x1; l1_page_table.entries[l1_index].l1_section_1m.shared = perms_attrs.shared; l1_page_table.entries[l1_index].l1_section_1m.not_global = perms_attrs.not_global; l1_page_table.entries[l1_index].l1_section_1m.zero = 0; l1_page_table.entries[l1_index].l1_section_1m.non_sec = perms_attrs.non_sec; l1_page_table.entries[l1_index].l1_section_1m.base_address = (pa >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT); } /** * @brief Converts a L1 1 MB section mapping to a full L2 table * When this function is called, something has happened that shouldn't * happen for the sake of run-time performance and determinism: the * attributes and/or permissions of a subset of a 1 MB memory range * currently represented by a level 1 page table entry of type 'section' * shall be modified so that they differ from the rest of the 1 MB * range's attributes/permissions. Therefore, the single L1 page table * entry has to be broken down to the full 256 4k-wide entries of a * L2 page table with identical properties so that afterwards, the * modification of the subset can be performed with a 4k granularity. * The risk at this point is that all L2 tables are already in use, * which will result in an assertion failure in the first contained * #arm_mmu_l2_map_page() call. * @warning While the conversion is being performed, interrupts are * locked globally and the MMU is disabled (the required * Zephyr code & data are still accessible in this state as * those are identity mapped). Expect non-deterministic be- * haviour / interrupt latencies while the conversion is in * progress! * * @param va 32-bit virtual address within the 1 MB range that shall * be converted from L1 1 MB section mapping to L2 4 kB page * mappings. * @param l2_page_table Pointer to an empty L2 page table allocated * for the purpose of replacing the L1 section * mapping. */ static void arm_mmu_remap_l1_section_to_l2_table(uint32_t va, struct arm_mmu_l2_page_table *l2_page_table) { struct arm_mmu_perms_attrs perms_attrs = {0}; uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK; uint32_t rem_size = MB(1); uint32_t reg_val; int lock_key; /* * Extract the permissions and attributes from the current 1 MB section entry. * This data will be carried over to the resulting L2 page table. */ perms_attrs.acc_perms = (l1_page_table.entries[l1_index].l1_section_1m.acc_perms2 << 1) | ((l1_page_table.entries[l1_index].l1_section_1m.acc_perms10 >> 1) & 0x1); perms_attrs.bufferable = l1_page_table.entries[l1_index].l1_section_1m.bufferable; perms_attrs.cacheable = l1_page_table.entries[l1_index].l1_section_1m.cacheable; perms_attrs.domain = l1_page_table.entries[l1_index].l1_section_1m.domain; perms_attrs.id_mask = (l1_page_table.entries[l1_index].l1_section_1m.id == ARM_MMU_PTE_ID_INVALID) ? 0x0 : 0x3; perms_attrs.not_global = l1_page_table.entries[l1_index].l1_section_1m.not_global; perms_attrs.non_sec = l1_page_table.entries[l1_index].l1_section_1m.non_sec; perms_attrs.shared = l1_page_table.entries[l1_index].l1_section_1m.shared; perms_attrs.tex = l1_page_table.entries[l1_index].l1_section_1m.tex; perms_attrs.exec_never = l1_page_table.entries[l1_index].l1_section_1m.exec_never; /* * Disable interrupts - no interrupts shall occur before the L2 table has * been set up in place of the former L1 section entry. */ lock_key = arch_irq_lock(); /* * Disable the MMU. The L1 PTE array and the L2 PT array may actually be * covered by the L1 PTE we're about to replace, so access to this data * must remain functional during the entire remap process. Yet, the only * memory areas for which L1 1 MB section entries are even considered are * those belonging to the Zephyr image. Those areas are *always* identity * mapped, so the MMU can be turned off and the relevant data will still * be available. */ reg_val = __get_SCTLR(); __set_SCTLR(reg_val & (~ARM_MMU_SCTLR_MMU_ENABLE_BIT)); /* * Clear the entire L1 PTE & re-configure it as a L2 PT reference * -> already sets the correct values for: zero0, zero1, impl_def. */ l1_page_table.entries[l1_index].word = 0; l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT; l1_page_table.entries[l1_index].l2_page_table_ref.domain = perms_attrs.domain; l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = perms_attrs.non_sec; l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address = (((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) & ARM_MMU_PT_L2_ADDR_MASK); /* Align the target VA to the base address of the section we're converting */ va &= ~(MB(1) - 1); while (rem_size > 0) { arm_mmu_l2_map_page(va, va, perms_attrs); rem_size -= KB(4); va += KB(4); } /* Remap complete, re-enable the MMU, unlock the interrupts. */ invalidate_tlb_all(); __set_SCTLR(reg_val); arch_irq_unlock(lock_key); } /** * @brief Maps a 4 kB memory page using a L2 page table entry * Maps a single 4 kB page of memory from the specified physical * address to the specified virtual address, using the provided * attributes and permissions which have already been converted * from the system's format provided to arch_mem_map() to the * bits / bit masks used in the L2 page table entry. * * @param va 32-bit target virtual address. * @param pa 32-bit physical address. * @param perms_attrs Permission and attribute bits in the format * used in the MMU's L2 page table entries. */ static void arm_mmu_l2_map_page(uint32_t va, uint32_t pa, struct arm_mmu_perms_attrs perms_attrs) { struct arm_mmu_l2_page_table *l2_page_table = NULL; uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK; uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) & ARM_MMU_PTE_L2_INDEX_MASK; /* * Use the calculated L1 index in order to determine if a L2 page * table is required in order to complete the current mapping. * -> See below for an explanation of the possible scenarios. */ if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_INVALID || (l1_page_table.entries[l1_index].undefined.id & ARM_MMU_PTE_ID_SECTION) != 0) { l2_page_table = arm_mmu_assign_l2_table(pa); __ASSERT(l2_page_table != NULL, "Unexpected L2 page table NULL pointer for VA 0x%08X", va); } /* * Check what is currently present at the corresponding L1 table entry. * The following scenarios are possible: * 1) The L1 PTE's ID bits are zero, as is the rest of the entry. * In this case, the L1 PTE is currently unused. A new L2 PT to * refer to in this entry has already been allocated above. * 2) The L1 PTE's ID bits indicate a L2 PT reference entry (01). * The corresponding L2 PT's address will be resolved using this * entry. * 3) The L1 PTE's ID bits may or may not be zero, and the rest of * the descriptor contains some non-zero data. This always indicates * an existing 1 MB section entry in this place. Checking only the * ID bits wouldn't be enough, as the only way to indicate a section * with neither R nor W permissions is to set the ID bits to 00 in * the AP[2:1] permissions model. As we're now about to map a single * page overlapping with the 1 MB section, the section has to be * converted into a L2 table. Afterwards, the current page mapping * can be added/modified. */ if (l1_page_table.entries[l1_index].word == 0) { /* The matching L1 PT entry is currently unused */ l1_page_table.entries[l1_index].l2_page_table_ref.id = ARM_MMU_PTE_ID_L2_PT; l1_page_table.entries[l1_index].l2_page_table_ref.zero0 = 0; l1_page_table.entries[l1_index].l2_page_table_ref.zero1 = 0; l1_page_table.entries[l1_index].l2_page_table_ref.impl_def = 0; l1_page_table.entries[l1_index].l2_page_table_ref.domain = 0; /* TODO */ l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = perms_attrs.non_sec; l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address = (((uint32_t)l2_page_table >> ARM_MMU_PT_L2_ADDR_SHIFT) & ARM_MMU_PT_L2_ADDR_MASK); } else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) { /* The matching L1 PT entry already points to a L2 PT */ l2_page_table = (struct arm_mmu_l2_page_table *) ((l1_page_table.entries[l1_index].word & (ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT))); /* * The only configuration bit contained in the L2 PT entry is the * NS bit. Set it according to the attributes passed to this function, * warn if there is a mismatch between the current page's NS attribute * value and the value currently contained in the L2 PT entry. */ if (l1_page_table.entries[l1_index].l2_page_table_ref.non_sec != perms_attrs.non_sec) { LOG_WRN("NS bit mismatch in L2 PT reference at L1 index [%u], " "re-configuring from %u to %u", l1_index, l1_page_table.entries[l1_index].l2_page_table_ref.non_sec, perms_attrs.non_sec); l1_page_table.entries[l1_index].l2_page_table_ref.non_sec = perms_attrs.non_sec; } } else if (l1_page_table.entries[l1_index].undefined.reserved != 0) { /* * The matching L1 PT entry currently holds a 1 MB section entry * in order to save a L2 table (as it's neither completely blank * nor a L2 PT reference), but now we have to map an overlapping * 4 kB page, so the section entry must be converted to a L2 table * first before the individual L2 entry for the page to be mapped is * accessed. A blank L2 PT has already been assigned above. */ arm_mmu_remap_l1_section_to_l2_table(va, l2_page_table); } /* * If the matching L2 PTE is blank, increment the number of used entries * in the L2 table. If the L2 PTE already contains some data, we're re- * placing the entry's data instead, the used entry count remains unchanged. * Once again, checking the ID bits might be misleading if the PTE declares * a page which has neither R nor W permissions. */ if (l2_page_table->entries[l2_index].word == 0) { arm_mmu_inc_l2_table_entries(l2_page_table); } l2_page_table->entries[l2_index].l2_page_4k.id = (ARM_MMU_PTE_ID_SMALL_PAGE & perms_attrs.id_mask); l2_page_table->entries[l2_index].l2_page_4k.id |= perms_attrs.exec_never; /* XN in [0] */ l2_page_table->entries[l2_index].l2_page_4k.bufferable = perms_attrs.bufferable; l2_page_table->entries[l2_index].l2_page_4k.cacheable = perms_attrs.cacheable; l2_page_table->entries[l2_index].l2_page_4k.acc_perms10 = ((perms_attrs.acc_perms & 0x1) << 1) | 0x1; l2_page_table->entries[l2_index].l2_page_4k.tex = perms_attrs.tex; l2_page_table->entries[l2_index].l2_page_4k.acc_perms2 = ((perms_attrs.acc_perms >> 1) & 0x1); l2_page_table->entries[l2_index].l2_page_4k.shared = perms_attrs.shared; l2_page_table->entries[l2_index].l2_page_4k.not_global = perms_attrs.not_global; l2_page_table->entries[l2_index].l2_page_4k.pa_base = ((pa >> ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT) & ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_MASK); } /** * @brief Unmaps a 4 kB memory page by clearing its L2 page table entry * Unmaps a single 4 kB page of memory from the specified virtual * address by clearing its respective L2 page table entry. * * @param va 32-bit virtual address to be unmapped. */ static void arm_mmu_l2_unmap_page(uint32_t va) { struct arm_mmu_l2_page_table *l2_page_table; uint32_t l1_index = (va >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK; uint32_t l2_index = (va >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) & ARM_MMU_PTE_L2_INDEX_MASK; if (l1_page_table.entries[l1_index].undefined.id != ARM_MMU_PTE_ID_L2_PT) { /* * No L2 PT currently exists for the given VA - this should be * tolerated without an error, just as in the case that while * a L2 PT exists, the corresponding PTE is blank - see explanation * below, the same applies here. */ return; } l2_page_table = (struct arm_mmu_l2_page_table *) ((l1_page_table.entries[l1_index].word & (ARM_MMU_PT_L2_ADDR_MASK << ARM_MMU_PT_L2_ADDR_SHIFT))); if (l2_page_table->entries[l2_index].word == 0) { /* * We're supposed to unmap a page at the given VA, but there currently * isn't anything mapped at this address, the L2 PTE is blank. * -> This is normal if a memory area is being mapped via k_mem_map, * which contains two calls to arch_mem_unmap (which effectively end up * here) in order to unmap the leading and trailing guard pages. * Therefore, it has to be expected that unmap calls are made for unmapped * memory which hasn't been in use before. * -> Just return, don't decrement the entry counter of the corresponding * L2 page table, as we're not actually clearing any PTEs. */ return; } if ((l2_page_table->entries[l2_index].undefined.id & ARM_MMU_PTE_ID_SMALL_PAGE) != ARM_MMU_PTE_ID_SMALL_PAGE) { LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid " "page table entry type in level 2 page table at " "L1 index [%u], L2 index [%u]", va, l1_index, l2_index); return; } l2_page_table->entries[l2_index].word = 0; arm_mmu_dec_l2_table_entries(l2_page_table); } /** * @brief MMU boot-time initialization function * Initializes the MMU at boot time. Sets up the page tables and * applies any specified memory mappings for either the different * sections of the Zephyr binary image, or for device memory as * specified at the SoC level. * * @retval Always 0, errors are handled by assertions. */ int z_arm_mmu_init(void) { uint32_t mem_range; uint32_t pa; uint32_t va; uint32_t attrs; uint32_t pt_attrs = 0; uint32_t rem_size; uint32_t reg_val = 0; struct arm_mmu_perms_attrs perms_attrs; __ASSERT(KB(4) == CONFIG_MMU_PAGE_SIZE, "MMU_PAGE_SIZE value %u is invalid, only 4 kB pages are supported\n", CONFIG_MMU_PAGE_SIZE); /* Set up the memory regions pre-defined by the image */ for (mem_range = 0; mem_range < ARRAY_SIZE(mmu_zephyr_ranges); mem_range++) { pa = mmu_zephyr_ranges[mem_range].start; rem_size = mmu_zephyr_ranges[mem_range].end - pa; attrs = mmu_zephyr_ranges[mem_range].attrs; perms_attrs = arm_mmu_convert_attr_flags(attrs); /* * Check if the L1 page table is within the region currently * being mapped. If so, store the permissions and attributes * of the current section. This information is required when * writing to the TTBR0 register. */ if (((uint32_t)&l1_page_table >= pa) && ((uint32_t)&l1_page_table < (pa + rem_size))) { pt_attrs = attrs; } while (rem_size > 0) { if (rem_size >= MB(1) && (pa & 0xFFFFF) == 0 && (attrs & MATTR_MAY_MAP_L1_SECTION)) { /* * Remaining area size > 1 MB & matching alignment * -> map a 1 MB section instead of individual 4 kB * pages with identical configuration. */ arm_mmu_l1_map_section(pa, pa, perms_attrs); rem_size -= MB(1); pa += MB(1); } else { arm_mmu_l2_map_page(pa, pa, perms_attrs); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; pa += KB(4); } } } /* Set up the memory regions defined at the SoC level */ for (mem_range = 0; mem_range < mmu_config.num_regions; mem_range++) { pa = (uint32_t)(mmu_config.mmu_regions[mem_range].base_pa); va = (uint32_t)(mmu_config.mmu_regions[mem_range].base_va); rem_size = (uint32_t)(mmu_config.mmu_regions[mem_range].size); attrs = mmu_config.mmu_regions[mem_range].attrs; perms_attrs = arm_mmu_convert_attr_flags(attrs); while (rem_size > 0) { arm_mmu_l2_map_page(va, pa, perms_attrs); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); pa += KB(4); } } /* Clear TTBR1 */ __asm__ __volatile__("mcr p15, 0, %0, c2, c0, 1" : : "r"(reg_val)); /* Write TTBCR: EAE, security not yet relevant, N[2:0] = 0 */ __asm__ __volatile__("mcr p15, 0, %0, c2, c0, 2" : : "r"(reg_val)); /* Write TTBR0 */ reg_val = ((uint32_t)&l1_page_table.entries[0] & ~0x3FFF); /* * Set IRGN, RGN, S in TTBR0 based on the configuration of the * memory area the actual page tables are located in. */ if (pt_attrs & MATTR_SHARED) { reg_val |= ARM_MMU_TTBR_SHAREABLE_BIT; } if (pt_attrs & MATTR_CACHE_OUTER_WB_WA) { reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_WA_CACHEABLE << ARM_MMU_TTBR_RGN_SHIFT); } else if (pt_attrs & MATTR_CACHE_OUTER_WT_nWA) { reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WT_CACHEABLE << ARM_MMU_TTBR_RGN_SHIFT); } else if (pt_attrs & MATTR_CACHE_OUTER_WB_nWA) { reg_val |= (ARM_MMU_TTBR_RGN_OUTER_WB_nWA_CACHEABLE << ARM_MMU_TTBR_RGN_SHIFT); } if (pt_attrs & MATTR_CACHE_INNER_WB_WA) { reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY; } else if (pt_attrs & MATTR_CACHE_INNER_WT_nWA) { reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY; } else if (pt_attrs & MATTR_CACHE_INNER_WB_nWA) { reg_val |= ARM_MMU_TTBR_IRGN0_BIT_MP_EXT_ONLY; reg_val |= ARM_MMU_TTBR_IRGN1_BIT_MP_EXT_ONLY; } __set_TTBR0(reg_val); /* Write DACR -> all domains to client = 01b. */ reg_val = ARM_MMU_DACR_ALL_DOMAINS_CLIENT; __set_DACR(reg_val); invalidate_tlb_all(); /* Enable the MMU and Cache in SCTLR */ reg_val = __get_SCTLR(); reg_val |= ARM_MMU_SCTLR_AFE_BIT; reg_val |= ARM_MMU_SCTLR_ICACHE_ENABLE_BIT; reg_val |= ARM_MMU_SCTLR_DCACHE_ENABLE_BIT; reg_val |= ARM_MMU_SCTLR_MMU_ENABLE_BIT; __set_SCTLR(reg_val); return 0; } /** * @brief ARMv7-specific implementation of memory mapping at run-time * Maps memory according to the parameters provided by the caller * at run-time. * * @param virt 32-bit target virtual address. * @param phys 32-bit physical address. * @param size Size (in bytes) of the memory area to map. * @param flags Memory attributes & permissions. Comp. K_MEM_... * flags in kernel/mm.h. * @retval 0 on success, -EINVAL if an invalid parameter is detected. */ static int __arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) { uint32_t va = (uint32_t)virt; uint32_t pa = (uint32_t)phys; uint32_t rem_size = (uint32_t)size; uint32_t conv_flags = MPERM_R; struct arm_mmu_perms_attrs perms_attrs; int key; if (size == 0) { LOG_ERR("Cannot map physical memory at 0x%08X: invalid " "zero size", (uint32_t)phys); return -EINVAL; } switch (flags & K_MEM_CACHE_MASK) { case K_MEM_CACHE_NONE: default: conv_flags |= MT_DEVICE; break; case K_MEM_CACHE_WB: conv_flags |= MT_NORMAL; conv_flags |= MATTR_SHARED; if (flags & K_MEM_PERM_RW) { conv_flags |= MATTR_CACHE_OUTER_WB_WA; conv_flags |= MATTR_CACHE_INNER_WB_WA; } else { conv_flags |= MATTR_CACHE_OUTER_WB_nWA; conv_flags |= MATTR_CACHE_INNER_WB_nWA; } break; case K_MEM_CACHE_WT: conv_flags |= MT_NORMAL; conv_flags |= MATTR_SHARED; conv_flags |= MATTR_CACHE_OUTER_WT_nWA; conv_flags |= MATTR_CACHE_INNER_WT_nWA; break; } if (flags & K_MEM_PERM_RW) { conv_flags |= MPERM_W; } if (flags & K_MEM_PERM_EXEC) { conv_flags |= MPERM_X; } perms_attrs = arm_mmu_convert_attr_flags(conv_flags); key = arch_irq_lock(); while (rem_size > 0) { arm_mmu_l2_map_page(va, pa, perms_attrs); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); pa += KB(4); } arch_irq_unlock(key); return 0; } /** * @brief Arch-specific wrapper function for memory mapping at run-time * Maps memory according to the parameters provided by the caller * at run-time. This function wraps the ARMv7 MMU specific implementation * #__arch_mem_map() for the upper layers of the memory management. * If the map operation fails, a kernel panic will be triggered. * * @param virt 32-bit target virtual address. * @param phys 32-bit physical address. * @param size Size (in bytes) of the memory area to map. * @param flags Memory attributes & permissions. Comp. K_MEM_... * flags in kernel/mm.h. */ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) { int ret = __arch_mem_map(virt, phys, size, flags); if (ret) { LOG_ERR("__arch_mem_map() returned %d", ret); k_panic(); } else { invalidate_tlb_all(); } } /** * @brief ARMv7-specific implementation of memory unmapping at run-time * Unmaps memory according to the parameters provided by the caller * at run-time. * * @param addr 32-bit virtual address to unmap. * @param size Size (in bytes) of the memory area to unmap. * @retval 0 on success, -EINVAL if an invalid parameter is detected. */ static int __arch_mem_unmap(void *addr, size_t size) { uint32_t va = (uint32_t)addr; uint32_t rem_size = (uint32_t)size; int key; if (addr == NULL) { LOG_ERR("Cannot unmap virtual memory: invalid NULL pointer"); return -EINVAL; } if (size == 0) { LOG_ERR("Cannot unmap virtual memory at 0x%08X: invalid " "zero size", (uint32_t)addr); return -EINVAL; } key = arch_irq_lock(); while (rem_size > 0) { arm_mmu_l2_unmap_page(va); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); } arch_irq_unlock(key); return 0; } /** * @brief Arch-specific wrapper function for memory unmapping at run-time * Unmaps memory according to the parameters provided by the caller * at run-time. This function wraps the ARMv7 MMU specific implementation * #__arch_mem_unmap() for the upper layers of the memory management. * * @param addr 32-bit virtual address to unmap. * @param size Size (in bytes) of the memory area to unmap. */ void arch_mem_unmap(void *addr, size_t size) { int ret = __arch_mem_unmap(addr, size); if (ret) { LOG_ERR("__arch_mem_unmap() returned %d", ret); } else { invalidate_tlb_all(); } } /** * @brief Arch-specific virtual-to-physical address resolver function * ARMv7 MMU specific implementation of a function that resolves the * physical address corresponding to the given virtual address. * * @param virt 32-bit target virtual address to resolve. * @param phys Pointer to a variable to which the resolved physical * address will be written. May be NULL if this information * is not actually required by the caller. * @retval 0 if the physical address corresponding to the specified * virtual address could be resolved successfully, -EFAULT * if the specified virtual address is not currently mapped. */ int arch_page_phys_get(void *virt, uintptr_t *phys) { uint32_t l1_index = ((uint32_t)virt >> ARM_MMU_PTE_L1_INDEX_PA_SHIFT) & ARM_MMU_PTE_L1_INDEX_MASK; uint32_t l2_index = ((uint32_t)virt >> ARM_MMU_PTE_L2_INDEX_PA_SHIFT) & ARM_MMU_PTE_L2_INDEX_MASK; struct arm_mmu_l2_page_table *l2_page_table; uint32_t pa_resolved = 0; uint32_t l2_pt_resolved; int rc = 0; int key; key = arch_irq_lock(); if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_SECTION) { /* * If the virtual address points to a level 1 PTE whose ID bits * identify it as a 1 MB section entry rather than a level 2 PT * entry, the given VA belongs to a memory region used by the * Zephyr image itself - it is only for those static regions that * L1 Section entries are used to save L2 tables if a sufficient- * ly large block of memory is specified. The memory regions be- * longing to the Zephyr image are identity mapped -> just return * the value of the VA as the value of the PA. */ pa_resolved = (uint32_t)virt; } else if (l1_page_table.entries[l1_index].undefined.id == ARM_MMU_PTE_ID_L2_PT) { /* * The VA points to a level 1 PTE which re-directs to a level 2 * PT. -> Assemble the level 2 PT pointer and resolve the PA for * the specified VA from there. */ l2_pt_resolved = l1_page_table.entries[l1_index].l2_page_table_ref.l2_page_table_address; l2_pt_resolved <<= ARM_MMU_PT_L2_ADDR_SHIFT; l2_page_table = (struct arm_mmu_l2_page_table *)l2_pt_resolved; /* * Check if the PTE for the specified VA is actually in use before * assembling & returning the corresponding PA. k_mem_unmap will * call this function for the leading & trailing guard pages when * unmapping a VA. As those guard pages were explicitly unmapped * when the VA was originally mapped, their L2 PTEs will be empty. * In that case, the return code of this function must not be 0. */ if (l2_page_table->entries[l2_index].word == 0) { rc = -EFAULT; } pa_resolved = l2_page_table->entries[l2_index].l2_page_4k.pa_base; pa_resolved <<= ARM_MMU_PTE_L2_SMALL_PAGE_ADDR_SHIFT; pa_resolved |= ((uint32_t)virt & ARM_MMU_ADDR_BELOW_PAGE_GRAN_MASK); } else { /* The level 1 PTE is invalid -> the specified VA is not mapped */ rc = -EFAULT; } arch_irq_unlock(key); if (phys) { *phys = (uintptr_t)pa_resolved; } return rc; }