From e88afd2c37017b7f8fefdd9b0a086cd11c1bba71 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 15 Jul 2021 13:15:29 -0700 Subject: [PATCH] kernel: mmu: pin/unpin boot sections during boot process During boot process, the boot sections need to be pinned in memory to prevent them from being paged out (to avoid pages being paged out and immediately paged in again). Once the boot process is completed (just before calling main()), the boot sections can be unpinned so the memory can be used for demand paging for paging in data pages. Signed-off-by: Daniel Leung --- kernel/include/kernel_internal.h | 5 +++ kernel/init.c | 4 +++ kernel/mmu.c | 62 +++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/kernel/include/kernel_internal.h b/kernel/include/kernel_internal.h index 684d87dc07c..dabee2222c4 100644 --- a/kernel/include/kernel_internal.h +++ b/kernel/include/kernel_internal.h @@ -217,6 +217,11 @@ void z_thread_mark_switched_out(void); */ void z_mem_manage_init(void); +/** + * @brief Finalize page frame management at the end of boot process. + */ +void z_mem_manage_boot_finish(void); + #define LOCKED(lck) for (k_spinlock_key_t __i = {}, \ __key = k_spin_lock(lck); \ !__i.key; \ diff --git a/kernel/init.c b/kernel/init.c index 8ca970d1f72..ddf8c3dc285 100644 --- a/kernel/init.c +++ b/kernel/init.c @@ -216,6 +216,10 @@ static void bg_thread_main(void *unused1, void *unused2, void *unused3) z_sys_init_run_level(_SYS_INIT_LEVEL_SMP); #endif +#ifdef CONFIG_MMU + z_mem_manage_boot_finish(); +#endif /* CONFIG_MMU */ + extern void main(void); main(); diff --git a/kernel/mmu.c b/kernel/mmu.c index 5e629a7bfb5..4d235858a92 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -722,6 +722,33 @@ size_t k_mem_region_align(uintptr_t *aligned_addr, size_t *aligned_size, return addr_offset; } +#if defined(CONFIG_LINKER_USE_BOOT_SECTION) || defined(CONFIG_LINKER_USE_PINNED_SECTION) +static void mark_linker_section_pinned(void *start_addr, void *end_addr, + bool pin) +{ + struct z_page_frame *pf; + uint8_t *addr; + + uintptr_t pinned_start = ROUND_DOWN(POINTER_TO_UINT(start_addr), + CONFIG_MMU_PAGE_SIZE); + uintptr_t pinned_end = ROUND_UP(POINTER_TO_UINT(end_addr), + CONFIG_MMU_PAGE_SIZE); + size_t pinned_size = pinned_end - pinned_start; + + VIRT_FOREACH(UINT_TO_POINTER(pinned_start), pinned_size, addr) + { + pf = z_phys_to_page_frame(Z_BOOT_VIRT_TO_PHYS(addr)); + frame_mapped_set(pf, addr); + + if (pin) { + pf->flags |= Z_PAGE_FRAME_PINNED; + } else { + pf->flags &= ~Z_PAGE_FRAME_PINNED; + } + } +} +#endif /* CONFIG_LINKER_USE_BOOT_SECTION) || CONFIG_LINKER_USE_PINNED_SECTION */ + void z_mem_manage_init(void) { uintptr_t phys; @@ -731,6 +758,8 @@ void z_mem_manage_init(void) free_page_frame_list_init(); + ARG_UNUSED(addr); + #ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES /* If some page frames are unavailable for use as memory, arch * code will mark Z_PAGE_FRAME_RESERVED in their flags @@ -738,6 +767,7 @@ void z_mem_manage_init(void) arch_reserved_pages_update(); #endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ +#ifdef CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT /* All pages composing the Zephyr image are mapped at boot in a * predictable way. This can change at runtime. */ @@ -758,22 +788,18 @@ void z_mem_manage_init(void) */ pf->flags |= Z_PAGE_FRAME_PINNED; } +#endif /* CONFIG_LINKER_GENERIC_SECTIONS_PRESENT_AT_BOOT */ + +#ifdef CONFIG_LINKER_USE_BOOT_SECTION + /* Pin the boot section to prevent it from being swapped out during + * boot process. Will be un-pinned once boot process completes. + */ + mark_linker_section_pinned(lnkr_boot_start, lnkr_boot_end, true); +#endif #ifdef CONFIG_LINKER_USE_PINNED_SECTION /* Pin the page frames correspondng to the pinned symbols */ - uintptr_t pinned_start = ROUND_DOWN(POINTER_TO_UINT(lnkr_pinned_start), - CONFIG_MMU_PAGE_SIZE); - uintptr_t pinned_end = ROUND_UP(POINTER_TO_UINT(lnkr_pinned_end), - CONFIG_MMU_PAGE_SIZE); - size_t pinned_size = pinned_end - pinned_start; - - VIRT_FOREACH(UINT_TO_POINTER(pinned_start), pinned_size, addr) - { - pf = z_phys_to_page_frame(Z_BOOT_VIRT_TO_PHYS(addr)); - frame_mapped_set(pf, addr); - - pf->flags |= Z_PAGE_FRAME_PINNED; - } + mark_linker_section_pinned(lnkr_pinned_start, lnkr_pinned_end, true); #endif /* Any remaining pages that aren't mapped, reserved, or pinned get @@ -809,6 +835,16 @@ void z_mem_manage_init(void) #endif } +void z_mem_manage_boot_finish(void) +{ +#ifdef CONFIG_LINKER_USE_BOOT_SECTION + /* At the end of boot process, unpin the boot sections + * as they don't need to be in memory all the time anymore. + */ + mark_linker_section_pinned(lnkr_boot_start, lnkr_boot_end, false); +#endif +} + #ifdef CONFIG_DEMAND_PAGING #ifdef CONFIG_DEMAND_PAGING_STATS