diff --git a/include/zephyr/kernel/internal/sched_priq.h b/include/zephyr/kernel/internal/sched_priq.h deleted file mode 100644 index 62b0984b77f..00000000000 --- a/include/zephyr/kernel/internal/sched_priq.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2018 Intel Corporation - * - * SPDX-License-Identifier: Apache-2.0 - */ -#ifndef ZEPHYR_INCLUDE_SCHED_PRIQ_H_ -#define ZEPHYR_INCLUDE_SCHED_PRIQ_H_ - -#include -#include -#include - -/* Two abstractions are defined here for "thread priority queues". - * - * One is a "dumb" list implementation appropriate for systems with - * small numbers of threads and sensitive to code size. It is stored - * in sorted order, taking an O(N) cost every time a thread is added - * to the list. This corresponds to the way the original _wait_q_t - * abstraction worked and is very fast as long as the number of - * threads is small. - * - * The other is a balanced tree "fast" implementation with rather - * larger code size (due to the data structure itself, the code here - * is just stubs) and higher constant-factor performance overhead, but - * much better O(logN) scaling in the presence of large number of - * threads. - * - * Each can be used for either the wait_q or system ready queue, - * configurable at build time. - */ - -struct k_thread; - -struct k_thread *z_priq_dumb_best(sys_dlist_t *pq); -void z_priq_dumb_remove(sys_dlist_t *pq, struct k_thread *thread); - -struct _priq_rb { - struct rbtree tree; - int next_order_key; -}; - -void z_priq_rb_add(struct _priq_rb *pq, struct k_thread *thread); -void z_priq_rb_remove(struct _priq_rb *pq, struct k_thread *thread); -struct k_thread *z_priq_rb_best(struct _priq_rb *pq); - -/* Traditional/textbook "multi-queue" structure. Separate lists for a - * small number (max 32 here) of fixed priorities. This corresponds - * to the original Zephyr scheduler. RAM requirements are - * comparatively high, but performance is very fast. Won't work with - * features like deadline scheduling which need large priority spaces - * to represent their requirements. - */ -struct _priq_mq { - sys_dlist_t queues[32]; - unsigned int bitmask; /* bit 1< #include #include -#include #include #include #include diff --git a/include/zephyr/kernel_structs.h b/include/zephyr/kernel_structs.h index 986165006eb..093fcd0a188 100644 --- a/include/zephyr/kernel_structs.h +++ b/include/zephyr/kernel_structs.h @@ -23,13 +23,13 @@ #if !defined(_ASMLANGUAGE) #include #include -#include #include #include #include #include #include #include +#include #endif #ifdef __cplusplus @@ -84,6 +84,43 @@ extern "C" { #if !defined(_ASMLANGUAGE) +/* Two abstractions are defined here for "thread priority queues". + * + * One is a "dumb" list implementation appropriate for systems with + * small numbers of threads and sensitive to code size. It is stored + * in sorted order, taking an O(N) cost every time a thread is added + * to the list. This corresponds to the way the original _wait_q_t + * abstraction worked and is very fast as long as the number of + * threads is small. + * + * The other is a balanced tree "fast" implementation with rather + * larger code size (due to the data structure itself, the code here + * is just stubs) and higher constant-factor performance overhead, but + * much better O(logN) scaling in the presence of large number of + * threads. + * + * Each can be used for either the wait_q or system ready queue, + * configurable at build time. + */ + +struct _priq_rb { + struct rbtree tree; + int next_order_key; +}; + + +/* Traditional/textbook "multi-queue" structure. Separate lists for a + * small number (max 32 here) of fixed priorities. This corresponds + * to the original Zephyr scheduler. RAM requirements are + * comparatively high, but performance is very fast. Won't work with + * features like deadline scheduling which need large priority spaces + * to represent their requirements. + */ +struct _priq_mq { + sys_dlist_t queues[32]; + unsigned int bitmask; /* bit 1<waitq) } -#endif +#endif /* CONFIG_WAITQ_SCALABLE */ /* kernel timeout record */ - struct _timeout; typedef void (*_timeout_func_t)(struct _timeout *t); diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index cd9f39c9558..f519b545564 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -59,6 +59,7 @@ list(APPEND kernel_files mem_slab.c thread.c version.c + priority_queues.c sched.c ) diff --git a/kernel/include/priority_q.h b/kernel/include/priority_q.h new file mode 100644 index 00000000000..8c9a1a70836 --- /dev/null +++ b/kernel/include/priority_q.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_KERNEL_INCLUDE_PRIORITY_Q_H_ +#define ZEPHYR_KERNEL_INCLUDE_PRIORITY_Q_H_ + + +/* Dump Scheduling */ +#if defined(CONFIG_SCHED_DUMB) +#define _priq_run_add z_priq_dumb_add +#define _priq_run_remove z_priq_dumb_remove +# if defined(CONFIG_SCHED_CPU_MASK) +# define _priq_run_best _priq_dumb_mask_best +# else +# define _priq_run_best z_priq_dumb_best +# endif +/* Scalable Scheduling */ +#elif defined(CONFIG_SCHED_SCALABLE) +#define _priq_run_add z_priq_rb_add +#define _priq_run_remove z_priq_rb_remove +#define _priq_run_best z_priq_rb_best + /* Multi Queue Scheduling */ +#elif defined(CONFIG_SCHED_MULTIQ) +#define _priq_run_add z_priq_mq_add +#define _priq_run_remove z_priq_mq_remove +#define _priq_run_best z_priq_mq_best +static ALWAYS_INLINE void z_priq_mq_add(struct _priq_mq *pq, struct k_thread *thread); +static ALWAYS_INLINE void z_priq_mq_remove(struct _priq_mq *pq, struct k_thread *thread); +#endif + +/* Scalable Wait Queue */ +#if defined(CONFIG_WAITQ_SCALABLE) +#define z_priq_wait_add z_priq_rb_add +#define _priq_wait_remove z_priq_rb_remove +#define _priq_wait_best z_priq_rb_best +/* Dump Wait Queue */ +#elif defined(CONFIG_WAITQ_DUMB) +#define z_priq_wait_add z_priq_dumb_add +#define _priq_wait_remove z_priq_dumb_remove +#define _priq_wait_best z_priq_dumb_best +#endif + +/* Dumb Scheduling*/ +struct k_thread *z_priq_dumb_best(sys_dlist_t *pq); +void z_priq_dumb_remove(sys_dlist_t *pq, struct k_thread *thread); + +/* Scalable Scheduling */ +void z_priq_rb_add(struct _priq_rb *pq, struct k_thread *thread); +void z_priq_rb_remove(struct _priq_rb *pq, struct k_thread *thread); + +/* Multi Queue Scheduling */ +struct k_thread *z_priq_mq_best(struct _priq_mq *pq); +struct k_thread *z_priq_rb_best(struct _priq_rb *pq); + + +bool z_priq_rb_lessthan(struct rbnode *a, struct rbnode *b); + + +#ifdef CONFIG_SCHED_MULTIQ +# if (K_LOWEST_THREAD_PRIO - K_HIGHEST_THREAD_PRIO) > 31 +# error Too many priorities for multiqueue scheduler (max 32) +# endif + +static ALWAYS_INLINE void z_priq_mq_add(struct _priq_mq *pq, + struct k_thread *thread) +{ + int priority_bit = thread->base.prio - K_HIGHEST_THREAD_PRIO; + + sys_dlist_append(&pq->queues[priority_bit], &thread->base.qnode_dlist); + pq->bitmask |= BIT(priority_bit); +} + +static ALWAYS_INLINE void z_priq_mq_remove(struct _priq_mq *pq, + struct k_thread *thread) +{ + int priority_bit = thread->base.prio - K_HIGHEST_THREAD_PRIO; + + sys_dlist_remove(&thread->base.qnode_dlist); + if (sys_dlist_is_empty(&pq->queues[priority_bit])) { + pq->bitmask &= ~BIT(priority_bit); + } +} +#endif /* CONFIG_SCHED_MULTIQ */ +#endif /* ZEPHYR_KERNEL_INCLUDE_PRIORITY_Q_H_ */ diff --git a/kernel/include/wait_q.h b/kernel/include/wait_q.h index 0c44a89eb66..2203a036ad2 100644 --- a/kernel/include/wait_q.h +++ b/kernel/include/wait_q.h @@ -12,8 +12,8 @@ #include #include #include -#include #include +#include #ifdef __cplusplus extern "C" { diff --git a/kernel/priority_queues.c b/kernel/priority_queues.c new file mode 100644 index 00000000000..5de9cdccf84 --- /dev/null +++ b/kernel/priority_queues.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018,2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +void z_priq_dumb_remove(sys_dlist_t *pq, struct k_thread *thread) +{ + ARG_UNUSED(pq); + + __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); + + sys_dlist_remove(&thread->base.qnode_dlist); +} + +struct k_thread *z_priq_dumb_best(sys_dlist_t *pq) +{ + struct k_thread *thread = NULL; + sys_dnode_t *n = sys_dlist_peek_head(pq); + + if (n != NULL) { + thread = CONTAINER_OF(n, struct k_thread, base.qnode_dlist); + } + return thread; +} + +bool z_priq_rb_lessthan(struct rbnode *a, struct rbnode *b) +{ + struct k_thread *thread_a, *thread_b; + int32_t cmp; + + thread_a = CONTAINER_OF(a, struct k_thread, base.qnode_rb); + thread_b = CONTAINER_OF(b, struct k_thread, base.qnode_rb); + + cmp = z_sched_prio_cmp(thread_a, thread_b); + + if (cmp > 0) { + return true; + } else if (cmp < 0) { + return false; + } else { + return thread_a->base.order_key < thread_b->base.order_key + ? 1 : 0; + } +} + +void z_priq_rb_add(struct _priq_rb *pq, struct k_thread *thread) +{ + struct k_thread *t; + + __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); + + thread->base.order_key = pq->next_order_key++; + + /* Renumber at wraparound. This is tiny code, and in practice + * will almost never be hit on real systems. BUT on very + * long-running systems where a priq never completely empties + * AND that contains very large numbers of threads, it can be + * a latency glitch to loop over all the threads like this. + */ + if (!pq->next_order_key) { + RB_FOR_EACH_CONTAINER(&pq->tree, t, base.qnode_rb) { + t->base.order_key = pq->next_order_key++; + } + } + + rb_insert(&pq->tree, &thread->base.qnode_rb); +} + +void z_priq_rb_remove(struct _priq_rb *pq, struct k_thread *thread) +{ + __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); + + rb_remove(&pq->tree, &thread->base.qnode_rb); + + if (!pq->tree.root) { + pq->next_order_key = 0; + } +} + +struct k_thread *z_priq_rb_best(struct _priq_rb *pq) +{ + struct k_thread *thread = NULL; + struct rbnode *n = rb_get_min(&pq->tree); + + if (n != NULL) { + thread = CONTAINER_OF(n, struct k_thread, base.qnode_rb); + } + return thread; +} + +struct k_thread *z_priq_mq_best(struct _priq_mq *pq) +{ + if (!pq->bitmask) { + return NULL; + } + + struct k_thread *thread = NULL; + sys_dlist_t *l = &pq->queues[__builtin_ctz(pq->bitmask)]; + sys_dnode_t *n = sys_dlist_peek_head(l); + + if (n != NULL) { + thread = CONTAINER_OF(n, struct k_thread, base.qnode_dlist); + } + return thread; +} diff --git a/kernel/sched.c b/kernel/sched.c index 13a7b14b8c8..15d9b1d6164 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -6,8 +6,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -22,38 +22,6 @@ LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); -#if defined(CONFIG_SCHED_DUMB) -#define _priq_run_add z_priq_dumb_add -#define _priq_run_remove z_priq_dumb_remove -# if defined(CONFIG_SCHED_CPU_MASK) -# define _priq_run_best _priq_dumb_mask_best -# else -# define _priq_run_best z_priq_dumb_best -# endif -#elif defined(CONFIG_SCHED_SCALABLE) -#define _priq_run_add z_priq_rb_add -#define _priq_run_remove z_priq_rb_remove -#define _priq_run_best z_priq_rb_best -#elif defined(CONFIG_SCHED_MULTIQ) -#define _priq_run_add z_priq_mq_add -#define _priq_run_remove z_priq_mq_remove -#define _priq_run_best z_priq_mq_best -static ALWAYS_INLINE void z_priq_mq_add(struct _priq_mq *pq, - struct k_thread *thread); -static ALWAYS_INLINE void z_priq_mq_remove(struct _priq_mq *pq, - struct k_thread *thread); -#endif - -#if defined(CONFIG_WAITQ_SCALABLE) -#define z_priq_wait_add z_priq_rb_add -#define _priq_wait_remove z_priq_rb_remove -#define _priq_wait_best z_priq_rb_best -#elif defined(CONFIG_WAITQ_DUMB) -#define z_priq_wait_add z_priq_dumb_add -#define _priq_wait_remove z_priq_dumb_remove -#define _priq_wait_best z_priq_dumb_best -#endif - struct k_spinlock sched_spinlock; static void update_cache(int preempt_ok); @@ -61,6 +29,7 @@ static void halt_thread(struct k_thread *thread, uint8_t new_state); static void add_to_waitq_locked(struct k_thread *thread, _wait_q_t *wait_q); + static inline int is_preempt(struct k_thread *thread) { /* explanation in kernel_struct.h */ @@ -1252,133 +1221,6 @@ void *z_get_next_switch_handle(void *interrupted) } #endif -void z_priq_dumb_remove(sys_dlist_t *pq, struct k_thread *thread) -{ - ARG_UNUSED(pq); - - __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); - - sys_dlist_remove(&thread->base.qnode_dlist); -} - -struct k_thread *z_priq_dumb_best(sys_dlist_t *pq) -{ - struct k_thread *thread = NULL; - sys_dnode_t *n = sys_dlist_peek_head(pq); - - if (n != NULL) { - thread = CONTAINER_OF(n, struct k_thread, base.qnode_dlist); - } - return thread; -} - -bool z_priq_rb_lessthan(struct rbnode *a, struct rbnode *b) -{ - struct k_thread *thread_a, *thread_b; - int32_t cmp; - - thread_a = CONTAINER_OF(a, struct k_thread, base.qnode_rb); - thread_b = CONTAINER_OF(b, struct k_thread, base.qnode_rb); - - cmp = z_sched_prio_cmp(thread_a, thread_b); - - if (cmp > 0) { - return true; - } else if (cmp < 0) { - return false; - } else { - return thread_a->base.order_key < thread_b->base.order_key - ? 1 : 0; - } -} - -void z_priq_rb_add(struct _priq_rb *pq, struct k_thread *thread) -{ - struct k_thread *t; - - __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); - - thread->base.order_key = pq->next_order_key++; - - /* Renumber at wraparound. This is tiny code, and in practice - * will almost never be hit on real systems. BUT on very - * long-running systems where a priq never completely empties - * AND that contains very large numbers of threads, it can be - * a latency glitch to loop over all the threads like this. - */ - if (!pq->next_order_key) { - RB_FOR_EACH_CONTAINER(&pq->tree, t, base.qnode_rb) { - t->base.order_key = pq->next_order_key++; - } - } - - rb_insert(&pq->tree, &thread->base.qnode_rb); -} - -void z_priq_rb_remove(struct _priq_rb *pq, struct k_thread *thread) -{ - __ASSERT_NO_MSG(!z_is_idle_thread_object(thread)); - - rb_remove(&pq->tree, &thread->base.qnode_rb); - - if (!pq->tree.root) { - pq->next_order_key = 0; - } -} - -struct k_thread *z_priq_rb_best(struct _priq_rb *pq) -{ - struct k_thread *thread = NULL; - struct rbnode *n = rb_get_min(&pq->tree); - - if (n != NULL) { - thread = CONTAINER_OF(n, struct k_thread, base.qnode_rb); - } - return thread; -} - -#ifdef CONFIG_SCHED_MULTIQ -# if (K_LOWEST_THREAD_PRIO - K_HIGHEST_THREAD_PRIO) > 31 -# error Too many priorities for multiqueue scheduler (max 32) -# endif - -static ALWAYS_INLINE void z_priq_mq_add(struct _priq_mq *pq, - struct k_thread *thread) -{ - int priority_bit = thread->base.prio - K_HIGHEST_THREAD_PRIO; - - sys_dlist_append(&pq->queues[priority_bit], &thread->base.qnode_dlist); - pq->bitmask |= BIT(priority_bit); -} - -static ALWAYS_INLINE void z_priq_mq_remove(struct _priq_mq *pq, - struct k_thread *thread) -{ - int priority_bit = thread->base.prio - K_HIGHEST_THREAD_PRIO; - - sys_dlist_remove(&thread->base.qnode_dlist); - if (sys_dlist_is_empty(&pq->queues[priority_bit])) { - pq->bitmask &= ~BIT(priority_bit); - } -} -#endif - -struct k_thread *z_priq_mq_best(struct _priq_mq *pq) -{ - if (!pq->bitmask) { - return NULL; - } - - struct k_thread *thread = NULL; - sys_dlist_t *l = &pq->queues[__builtin_ctz(pq->bitmask)]; - sys_dnode_t *n = sys_dlist_peek_head(l); - - if (n != NULL) { - thread = CONTAINER_OF(n, struct k_thread, base.qnode_dlist); - } - return thread; -} - int z_unpend_all(_wait_q_t *wait_q) { int need_sched = 0;