kernel: mmu: collect more demand paging statistics
This adds more bits to gather statistics on demand paging, e.g. clean vs dirty pages evicted, # page faults with IRQ locked/unlocked, etc. Also extends this to gather per-thread demand paging statistics. Signed-off-by: Daniel Leung <daniel.leung@intel.com>
This commit is contained in:
parent
d5c3be3948
commit
ae86519819
7 changed files with 236 additions and 32 deletions
21
arch/Kconfig
21
arch/Kconfig
|
@ -648,7 +648,7 @@ config KERNEL_VM_SIZE
|
|||
implement a notion of "high" memory in Zephyr to work around physical
|
||||
RAM size larger than the defined bounds of the virtual address space.
|
||||
|
||||
config DEMAND_PAGING
|
||||
menuconfig DEMAND_PAGING
|
||||
bool "Enable demand paging [EXPERIMENTAL]"
|
||||
depends on ARCH_HAS_DEMAND_PAGING
|
||||
help
|
||||
|
@ -671,6 +671,25 @@ config DEMAND_PAGING_ALLOW_IRQ
|
|||
If this option is disabled, the page fault servicing logic
|
||||
runs with interrupts disabled for the entire operation. However,
|
||||
ISRs may also page fault.
|
||||
|
||||
config DEMAND_PAGING_STATS
|
||||
bool "Gather Demand Paging Statistics"
|
||||
help
|
||||
This enables gathering various statistics related to demand paging,
|
||||
e.g. number of pagefaults. This is useful for tuning eviction
|
||||
algorithms and optimizing backing store.
|
||||
|
||||
Should say N in production system as this is not without cost.
|
||||
|
||||
config DEMAND_PAGING_THREAD_STATS
|
||||
bool "Gather per Thread Demand Paging Statistics"
|
||||
depends on DEMAND_PAGING_STATS
|
||||
help
|
||||
This enables gathering per thread statistics related to demand
|
||||
paging.
|
||||
|
||||
Should say N in production system as this is not without cost.
|
||||
|
||||
endif # DEMAND_PAGING
|
||||
endif # MMU
|
||||
|
||||
|
|
|
@ -7,6 +7,10 @@
|
|||
#ifndef ZEPHYR_INCLUDE_KERNEL_THREAD_H_
|
||||
#define ZEPHYR_INCLUDE_KERNEL_THREAD_H_
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
#include <sys/mem_manage.h>
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @typedef k_thread_entry_t
|
||||
* @brief Thread entry point function type.
|
||||
|
@ -279,6 +283,11 @@ struct k_thread {
|
|||
struct _thread_runtime_stats rt_stats;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
/** Paging statistics */
|
||||
struct k_mem_paging_stats_t paging_stats;
|
||||
#endif
|
||||
|
||||
/** arch-specifics: must always be at the end */
|
||||
struct _thread_arch arch;
|
||||
};
|
||||
|
|
|
@ -79,6 +79,34 @@
|
|||
#include <inttypes.h>
|
||||
#include <sys/__assert.h>
|
||||
|
||||
struct k_mem_paging_stats_t {
|
||||
#ifdef CONFIG_DEMAND_PAGING_STATS
|
||||
struct {
|
||||
/** Number of page faults */
|
||||
unsigned long cnt;
|
||||
|
||||
/** Number of page faults with IRQ locked */
|
||||
unsigned long irq_locked;
|
||||
|
||||
/** Number of page faults with IRQ unlocked */
|
||||
unsigned long irq_unlocked;
|
||||
|
||||
#ifndef CONFIG_DEMAND_PAGING_ALLOW_IRQ
|
||||
/** Number of page faults while in ISR */
|
||||
unsigned long in_isr;
|
||||
#endif
|
||||
} pagefaults;
|
||||
|
||||
struct {
|
||||
/** Number of clean pages selected for eviction */
|
||||
unsigned long clean;
|
||||
|
||||
/** Number of dirty pages selected for eviction */
|
||||
unsigned long dirty;
|
||||
} eviction;
|
||||
#endif /* CONFIG_DEMAND_PAGING_STATS */
|
||||
};
|
||||
|
||||
/* Just like Z_MEM_PHYS_ADDR() but with type safety and assertions */
|
||||
static inline uintptr_t z_mem_phys_addr(void *virt)
|
||||
{
|
||||
|
@ -349,6 +377,36 @@ void k_mem_pin(void *addr, size_t size);
|
|||
void k_mem_unpin(void *addr, size_t size);
|
||||
#endif /* CONFIG_DEMAND_PAGING */
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_STATS
|
||||
/**
|
||||
* Get the paging statistics since system startup
|
||||
*
|
||||
* This populates the paging statistics struct being passed in
|
||||
* as argument.
|
||||
*
|
||||
* @param[in,out] stats Paging statistics struct to be filled.
|
||||
*/
|
||||
__syscall void k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats);
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
/**
|
||||
* Get the paging statistics since system startup for a thread
|
||||
*
|
||||
* This populates the paging statistics struct being passed in
|
||||
* as argument for a particular thread.
|
||||
*
|
||||
* @param[in] tid Thread ID
|
||||
* @param[in,out] stats Paging statistics struct to be filled.
|
||||
*/
|
||||
__syscall
|
||||
void k_mem_paging_thread_stats_get(k_tid_t tid,
|
||||
struct k_mem_paging_stats_t *stats);
|
||||
#endif /* CONFIG_DEMAND_PAGING_THREAD_STATS */
|
||||
|
||||
#include <syscalls/mem_manage.h>
|
||||
|
||||
#endif /* CONFIG_DEMAND_PAGING_STATS */
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -33,6 +33,11 @@ list(APPEND kernel_files
|
|||
xip.c)
|
||||
endif()
|
||||
|
||||
if(CONFIG_DEMAND_PAGING_STATS)
|
||||
list(APPEND kernel_files
|
||||
paging/statistics.c)
|
||||
endif()
|
||||
|
||||
add_library(kernel ${kernel_files})
|
||||
|
||||
# Kernel files has the macro __ZEPHYR_SUPERVISOR__ set so that it
|
||||
|
|
103
kernel/mmu.c
103
kernel/mmu.c
|
@ -12,6 +12,7 @@
|
|||
#include <mmu.h>
|
||||
#include <init.h>
|
||||
#include <kernel_internal.h>
|
||||
#include <syscall_handler.h>
|
||||
#include <linker/linker-defs.h>
|
||||
#include <logging/log.h>
|
||||
LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);
|
||||
|
@ -563,7 +564,10 @@ void z_mem_manage_init(void)
|
|||
}
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING
|
||||
static unsigned long z_num_pagefaults;
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_STATS
|
||||
struct k_mem_paging_stats_t paging_stats;
|
||||
#endif
|
||||
|
||||
/* Current implementation relies on interrupt locking to any prevent page table
|
||||
* access, which falls over if other CPUs are active. Addressing this is not
|
||||
|
@ -786,6 +790,65 @@ out:
|
|||
return ret;
|
||||
}
|
||||
|
||||
static inline void paging_stats_faults_inc(struct k_thread *faulting_thread,
|
||||
int key)
|
||||
{
|
||||
#ifdef CONFIG_DEMAND_PAGING_STATS
|
||||
bool is_irq_unlocked = arch_irq_unlocked(key);
|
||||
|
||||
paging_stats.pagefaults.cnt++;
|
||||
|
||||
if (is_irq_unlocked) {
|
||||
paging_stats.pagefaults.irq_unlocked++;
|
||||
} else {
|
||||
paging_stats.pagefaults.irq_locked++;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
faulting_thread->paging_stats.pagefaults.cnt++;
|
||||
|
||||
if (is_irq_unlocked) {
|
||||
faulting_thread->paging_stats.pagefaults.irq_unlocked++;
|
||||
} else {
|
||||
faulting_thread->paging_stats.pagefaults.irq_locked++;
|
||||
}
|
||||
#else
|
||||
ARG_UNUSED(faulting_thread);
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_DEMAND_PAGING_ALLOW_IRQ
|
||||
if (k_is_in_isr()) {
|
||||
paging_stats.pagefaults.in_isr++;
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
faulting_thread->paging_stats.pagefaults.in_isr++;
|
||||
#endif
|
||||
}
|
||||
#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */
|
||||
#endif /* CONFIG_DEMAND_PAGING_STATS */
|
||||
}
|
||||
|
||||
static inline void paging_stats_eviction_inc(struct k_thread *faulting_thread,
|
||||
bool dirty)
|
||||
{
|
||||
#ifdef CONFIG_DEMAND_PAGING_STATS
|
||||
if (dirty) {
|
||||
paging_stats.eviction.dirty++;
|
||||
} else {
|
||||
paging_stats.eviction.clean++;
|
||||
}
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
if (dirty) {
|
||||
faulting_thread->paging_stats.eviction.dirty++;
|
||||
} else {
|
||||
faulting_thread->paging_stats.eviction.clean++;
|
||||
}
|
||||
#else
|
||||
ARG_UNUSED(faulting_thread);
|
||||
#endif /* CONFIG_DEMAND_PAGING_THREAD_STATS */
|
||||
#endif /* CONFIG_DEMAND_PAGING_STATS */
|
||||
}
|
||||
|
||||
static bool do_page_fault(void *addr, bool pin)
|
||||
{
|
||||
struct z_page_frame *pf;
|
||||
|
@ -794,6 +857,7 @@ static bool do_page_fault(void *addr, bool pin)
|
|||
enum arch_page_location status;
|
||||
bool result;
|
||||
bool dirty = false;
|
||||
struct k_thread *faulting_thread = _current_cpu->current;
|
||||
|
||||
__ASSERT(page_frames_initialized, "page fault at %p happened too early",
|
||||
addr);
|
||||
|
@ -802,13 +866,7 @@ static bool do_page_fault(void *addr, bool pin)
|
|||
|
||||
/*
|
||||
* TODO: Add performance accounting:
|
||||
* - Number of pagefaults
|
||||
* * gathered on a per-thread basis:
|
||||
* . Pagefaults with IRQs locked in faulting thread (bad)
|
||||
* . Pagefaults with IRQs unlocked in faulting thread
|
||||
* * Pagefaults in ISRs (if allowed)
|
||||
* - z_eviction_select() metrics
|
||||
* * Clean vs dirty page eviction counts
|
||||
* * execution time histogram
|
||||
* * periodic timer execution time histogram (if implemented)
|
||||
* - z_backing_store_page_out() execution time histogram
|
||||
|
@ -853,6 +911,9 @@ static bool do_page_fault(void *addr, bool pin)
|
|||
goto out;
|
||||
}
|
||||
result = true;
|
||||
|
||||
paging_stats_faults_inc(faulting_thread, key);
|
||||
|
||||
if (status == ARCH_PAGE_LOCATION_PAGED_IN) {
|
||||
if (pin) {
|
||||
/* It's a physical memory address */
|
||||
|
@ -874,6 +935,8 @@ static bool do_page_fault(void *addr, bool pin)
|
|||
__ASSERT(pf != NULL, "failed to get a page frame");
|
||||
LOG_DBG("evicting %p at 0x%lx", pf->addr,
|
||||
z_page_frame_to_phys(pf));
|
||||
|
||||
paging_stats_eviction_inc(faulting_thread, dirty);
|
||||
}
|
||||
ret = page_frame_prepare_locked(pf, &dirty, true, &page_out_location);
|
||||
__ASSERT(ret == 0, "failed to prepare page frame");
|
||||
|
@ -946,30 +1009,7 @@ void k_mem_pin(void *addr, size_t size)
|
|||
|
||||
bool z_page_fault(void *addr)
|
||||
{
|
||||
bool ret;
|
||||
|
||||
ret = do_page_fault(addr, false);
|
||||
if (ret) {
|
||||
/* Wasn't an error, increment page fault count */
|
||||
int key;
|
||||
|
||||
key = irq_lock();
|
||||
z_num_pagefaults++;
|
||||
irq_unlock(key);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
unsigned long z_num_pagefaults_get(void)
|
||||
{
|
||||
unsigned long ret;
|
||||
int key;
|
||||
|
||||
key = irq_lock();
|
||||
ret = z_num_pagefaults;
|
||||
irq_unlock(key);
|
||||
|
||||
return ret;
|
||||
return do_page_fault(addr, false);
|
||||
}
|
||||
|
||||
static void do_mem_unpin(void *addr)
|
||||
|
@ -995,4 +1035,5 @@ void k_mem_unpin(void *addr, size_t size)
|
|||
addr);
|
||||
virt_region_foreach(addr, size, do_mem_unpin);
|
||||
}
|
||||
|
||||
#endif /* CONFIG_DEMAND_PAGING */
|
||||
|
|
71
kernel/paging/statistics.c
Normal file
71
kernel/paging/statistics.c
Normal file
|
@ -0,0 +1,71 @@
|
|||
/*
|
||||
* Copyright (c) 2021 Intel Corporation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <kernel.h>
|
||||
#include <kernel_internal.h>
|
||||
#include <syscall_handler.h>
|
||||
#include <toolchain.h>
|
||||
#include <sys/mem_manage.h>
|
||||
|
||||
extern struct k_mem_paging_stats_t paging_stats;
|
||||
|
||||
unsigned long z_num_pagefaults_get(void)
|
||||
{
|
||||
unsigned long ret;
|
||||
int key;
|
||||
|
||||
key = irq_lock();
|
||||
ret = paging_stats.pagefaults.cnt;
|
||||
irq_unlock(key);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void z_impl_k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats)
|
||||
{
|
||||
if (stats == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Copy statistics */
|
||||
memcpy(stats, &paging_stats, sizeof(paging_stats));
|
||||
}
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
static inline
|
||||
void z_vrfy_k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats)
|
||||
{
|
||||
Z_OOPS(Z_SYSCALL_MEMORY_WRITE(stats, sizeof(*stats)));
|
||||
z_impl_k_mem_paging_stats_get(stats);
|
||||
}
|
||||
#include <syscalls/k_mem_paging_stats_get_mrsh.c>
|
||||
#endif /* CONFIG_USERSPACE */
|
||||
|
||||
#ifdef CONFIG_DEMAND_PAGING_THREAD_STATS
|
||||
void z_impl_k_mem_paging_thread_stats_get(k_tid_t tid,
|
||||
struct k_mem_paging_stats_t *stats)
|
||||
{
|
||||
if ((tid == NULL) || (stats == NULL)) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Copy statistics */
|
||||
memcpy(stats, &tid->paging_stats, sizeof(tid->paging_stats));
|
||||
}
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
static inline
|
||||
void z_vrfy_k_mem_paging_thread_stats_get(k_tid_t tid,
|
||||
struct k_mem_paging_stats_t *stats)
|
||||
{
|
||||
Z_OOPS(Z_SYSCALL_OBJ(tid, K_OBJ_THREAD));
|
||||
Z_OOPS(Z_SYSCALL_MEMORY_WRITE(stats, sizeof(*stats)));
|
||||
z_impl_k_mem_paging_thread_stats_get(tid, stats);
|
||||
}
|
||||
#include <syscalls/k_mem_paging_thread_stats_get_mrsh.c>
|
||||
#endif /* CONFIG_USERSPACE */
|
||||
|
||||
#endif /* CONFIG_DEMAND_PAGING_THREAD_STATS */
|
|
@ -1 +1,2 @@
|
|||
CONFIG_ZTEST=y
|
||||
CONFIG_DEMAND_PAGING_STATS=y
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue