From 7830f87ccdf534cf0ff2e90a881df5da0f9e15e2 Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Thu, 7 Oct 2021 08:52:00 +0200 Subject: [PATCH] mmu: get virtual alignment from physical address On ARM64 platforms, when mapping multiple memory zones with size not multiple of a L2 block size (2MiB), all the following mappings will probably use L3 tables. And a huge mapping will consume all possible L3 tables. In order to reduce usage of L3 tables, this introduces a new arch_virt_region_align() optional architecture specific call to eventually return a more optimal virtual address alignment than the default MMU_PAGE_SIZE. This alignment is used in virt_region_alloc() by: - requesting more pages in virt_region_bitmap to make sure we request up to the possible aligned virtual address - freeing the supplementary pages used for alignment Suggested-by: Nicolas Pitre Signed-off-by: Neil Armstrong --- include/sys/arch_interface.h | 15 ++++ kernel/mmu.c | 139 +++++++++++++++++++++++++---------- 2 files changed, 117 insertions(+), 37 deletions(-) diff --git a/include/sys/arch_interface.h b/include/sys/arch_interface.h index 3a49deb90b4..6b280d8d1e5 100644 --- a/include/sys/arch_interface.h +++ b/include/sys/arch_interface.h @@ -649,6 +649,21 @@ void arch_mem_domain_partition_add(struct k_mem_domain *domain, */ int arch_buffer_validate(void *addr, size_t size, int write); +/** + * Get the optimal virtual region alignment to optimize the MMU table layout + * + * Some MMU HW requires some region to be aligned to some of the intermediate + * block alignment in order to reduce table usage. + * This call returns the optimal virtual address alignment in order to permit + * such optimization in the following MMU mapping call. + * + * @param[in] phys Physical address of region to be mapped, aligned to MMU_PAGE_SIZE + * @param[in] size Size of region to be mapped, aligned to MMU_PAGE_SIZE + * + * @retval alignment to apply on the virtual address of this region + */ +size_t arch_virt_region_align(uintptr_t phys, size_t size); + /** * Perform a one-way transition from supervisor to kernel mode. * diff --git a/kernel/mmu.c b/kernel/mmu.c index 72f8bf9c1e1..5e5f78b8a6f 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -226,40 +226,6 @@ static void virt_region_init(void) virt_region_inited = true; } -static void *virt_region_alloc(size_t size) -{ - uintptr_t dest_addr; - size_t offset; - size_t num_bits; - int ret; - - if (unlikely(!virt_region_inited)) { - virt_region_init(); - } - - num_bits = size / CONFIG_MMU_PAGE_SIZE; - ret = sys_bitarray_alloc(&virt_region_bitmap, num_bits, &offset); - if (ret != 0) { - LOG_ERR("insufficient virtual address space (requested %zu)", - size); - return NULL; - } - - /* Remember that bit #0 in bitmap corresponds to the highest - * virtual address. So here we need to go downwards (backwards?) - * to get the starting address of the allocated region. - */ - dest_addr = virt_from_bitmap_offset(offset, size); - - /* Need to make sure this does not step into kernel memory */ - if (dest_addr < POINTER_TO_UINT(Z_VIRT_REGION_START_ADDR)) { - (void)sys_bitarray_free(&virt_region_bitmap, size, offset); - return NULL; - } - - return UINT_TO_POINTER(dest_addr); -} - static void virt_region_free(void *vaddr, size_t size) { size_t offset, num_bits; @@ -282,6 +248,86 @@ static void virt_region_free(void *vaddr, size_t size) (void)sys_bitarray_free(&virt_region_bitmap, num_bits, offset); } +static void *virt_region_alloc(size_t size, size_t align) +{ + uintptr_t dest_addr; + size_t alloc_size; + size_t offset; + size_t num_bits; + int ret; + + if (unlikely(!virt_region_inited)) { + virt_region_init(); + } + + /* Possibly request more pages to ensure we can get an aligned virtual address */ + num_bits = (size + align - CONFIG_MMU_PAGE_SIZE) / CONFIG_MMU_PAGE_SIZE; + alloc_size = num_bits * CONFIG_MMU_PAGE_SIZE; + ret = sys_bitarray_alloc(&virt_region_bitmap, num_bits, &offset); + if (ret != 0) { + LOG_ERR("insufficient virtual address space (requested %zu)", + size); + return NULL; + } + + /* Remember that bit #0 in bitmap corresponds to the highest + * virtual address. So here we need to go downwards (backwards?) + * to get the starting address of the allocated region. + */ + dest_addr = virt_from_bitmap_offset(offset, alloc_size); + + if (alloc_size > size) { + uintptr_t aligned_dest_addr = ROUND_UP(dest_addr, align); + + /* Here is the memory organization when trying to get an aligned + * virtual address: + * + * +--------------+ <- Z_VIRT_RAM_START + * | Undefined VM | + * +--------------+ <- Z_KERNEL_VIRT_START (often == Z_VIRT_RAM_START) + * | Mapping for | + * | main kernel | + * | image | + * | | + * | | + * +--------------+ <- Z_FREE_VM_START + * | ... | + * +==============+ <- dest_addr + * | Unused | + * |..............| <- aligned_dest_addr + * | | + * | Aligned | + * | Mapping | + * | | + * |..............| <- aligned_dest_addr + size + * | Unused | + * +==============+ <- offset from Z_VIRT_RAM_END == dest_addr + alloc_size + * | ... | + * +--------------+ + * | Mapping | + * +--------------+ + * | Reserved | + * +--------------+ <- Z_VIRT_RAM_END + */ + + /* Free the two unused regions */ + virt_region_free(UINT_TO_POINTER(dest_addr), + aligned_dest_addr - dest_addr); + virt_region_free(UINT_TO_POINTER(aligned_dest_addr + size), + (dest_addr + alloc_size) - (aligned_dest_addr + size)); + + dest_addr = aligned_dest_addr; + } + + /* Need to make sure this does not step into kernel memory */ + if (dest_addr < POINTER_TO_UINT(Z_VIRT_REGION_START_ADDR)) { + (void)sys_bitarray_free(&virt_region_bitmap, size, offset); + return NULL; + } + + return UINT_TO_POINTER(dest_addr); +} + /* * Free page frames management * @@ -492,7 +538,7 @@ void *k_mem_map(size_t size, uint32_t flags) */ total_size = size + CONFIG_MMU_PAGE_SIZE * 2; - dst = virt_region_alloc(total_size); + dst = virt_region_alloc(total_size, CONFIG_MMU_PAGE_SIZE); if (dst == NULL) { /* Address space has no free region */ goto out; @@ -638,6 +684,23 @@ size_t k_mem_free_get(void) return ret * (size_t)CONFIG_MMU_PAGE_SIZE; } +/* Get the default virtual region alignment, here the default MMU page size + * + * @param[in] phys Physical address of region to be mapped, aligned to MMU_PAGE_SIZE + * @param[in] size Size of region to be mapped, aligned to MMU_PAGE_SIZE + * + * @retval alignment to apply on the virtual address of this region + */ +static size_t virt_region_align(uintptr_t phys, size_t size) +{ + ARG_UNUSED(phys); + ARG_UNUSED(size); + + return CONFIG_MMU_PAGE_SIZE; +} + +__weak FUNC_ALIAS(virt_region_align, arch_virt_region_align, size_t); + /* This may be called from arch early boot code before z_cstart() is invoked. * Data will be copied and BSS zeroed, but this must not rely on any * initialization functions being called prior to work correctly. @@ -645,7 +708,7 @@ size_t k_mem_free_get(void) void z_phys_map(uint8_t **virt_ptr, uintptr_t phys, size_t size, uint32_t flags) { uintptr_t aligned_phys, addr_offset; - size_t aligned_size; + size_t aligned_size, align_boundary; k_spinlock_key_t key; uint8_t *dest_addr; @@ -657,9 +720,11 @@ void z_phys_map(uint8_t **virt_ptr, uintptr_t phys, size_t size, uint32_t flags) "wraparound for physical address 0x%lx (size %zu)", aligned_phys, aligned_size); + align_boundary = arch_virt_region_align(aligned_phys, aligned_size); + key = k_spin_lock(&z_mm_lock); /* Obtain an appropriately sized chunk of virtual memory */ - dest_addr = virt_region_alloc(aligned_size); + dest_addr = virt_region_alloc(aligned_size, align_boundary); if (!dest_addr) { goto fail; }