kernel: add futex support
A k_futex is a lightweight mutual exclusion primitive designed to minimize kernel involvement. Uncontended operation relies only on atomic access to shared memory. k_futex structure lives in application memory. And when using futexes, the majority of the synchronization operations are performed in user mode. A user-mode thread employs the futex wait system call only when it is likely that the program has to block for a longer time until the condition becomes true. When the condition comes true, futex wake operation will be used to wake up one or more threads waiting on that futex. This patch implements two futex operations: k_futex_wait and k_futex_wake. For k_futex_wait, the comparison with the expected value, and starting to sleep are performed atomically to prevent lost wake-ups. If different context changed futex's value after the calling use-mode thread decided to block himself based on the old value, the comparison will help observing the value change and will not start to sleep. And for k_futex_wake, it will wake at most num_waiters of the waiters that are sleeping on that futex. But no guarantees are made on which threads are woken, that means scheduling priority is not taken into consideration. Fixes: #14493. Signed-off-by: Wentong Wu <wentong.wu@intel.com>
This commit is contained in:
parent
4e6e2e3e85
commit
5611e92347
5 changed files with 225 additions and 1 deletions
|
@ -129,6 +129,7 @@ struct k_poll_event;
|
|||
struct k_poll_signal;
|
||||
struct k_mem_domain;
|
||||
struct k_mem_partition;
|
||||
struct k_futex;
|
||||
|
||||
/* This enumeration needs to be kept in sync with the lists of kernel objects
|
||||
* and subsystems in scripts/gen_kobject_list.py, as well as the otype_to_str()
|
||||
|
@ -2088,6 +2089,94 @@ static inline void *z_impl_k_queue_peek_tail(struct k_queue *queue)
|
|||
|
||||
/** @} */
|
||||
|
||||
#ifdef CONFIG_USERSPACE
|
||||
/**
|
||||
* @brief futex structure
|
||||
*
|
||||
* A k_futex is a lightweight mutual exclusion primitive designed
|
||||
* to minimize kernel involvement. Uncontended operation relies
|
||||
* only on atomic access to shared memory. k_futex are tracked as
|
||||
* kernel objects and can live in user memory so any access bypass
|
||||
* the kernel object permission management mechanism.
|
||||
*/
|
||||
struct k_futex {
|
||||
atomic_t val;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief futex kernel data structure
|
||||
*
|
||||
* z_futex_data are the helper data structure for k_futex to complete
|
||||
* futex contended operation on kernel side, structure z_futex_data
|
||||
* of every futex object is invisible in user mode.
|
||||
*/
|
||||
struct z_futex_data {
|
||||
_wait_q_t wait_q;
|
||||
struct k_spinlock lock;
|
||||
};
|
||||
|
||||
#define Z_FUTEX_DATA_INITIALIZER(obj) \
|
||||
{ \
|
||||
.wait_q = Z_WAIT_Q_INIT(&obj.wait_q) \
|
||||
}
|
||||
|
||||
/**
|
||||
* @defgroup futex_apis FUTEX APIs
|
||||
* @ingroup kernel_apis
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Initialize a futex.
|
||||
*
|
||||
* This routine initializes a futex object, prior to its first use.
|
||||
*
|
||||
* @param futex Address of the k_futex.
|
||||
*
|
||||
* @return N/A
|
||||
*/
|
||||
__syscall void k_futex_init(struct k_futex *futex);
|
||||
|
||||
/**
|
||||
* @brief Pend the current thread on a futex
|
||||
*
|
||||
* Tests that the supplied futex contains the expected value, and if so,
|
||||
* goes to sleep until some other thread calls k_futex_wake() on it.
|
||||
*
|
||||
* @param futex Address of the futex.
|
||||
* @param expected Expected value of the futex, if it is different the caller
|
||||
* will not wait on it.
|
||||
* @param timeout Waiting period on the futex, in milliseconds, or one of the
|
||||
* special values K_NO_WAIT or K_FOREVER.
|
||||
* @retval -EACCES Caller does not have read access to futex address.
|
||||
* @retval -EAGAIN If the futex value did not match the expected parameter.
|
||||
* @retval -EINVAL Futex parameter address not recognized by the kernel.
|
||||
* @retval -ETIMEDOUT Thread woke up due to timeout and not a futex wakeup.
|
||||
* @retval 0 if the caller went to sleep and was woken up. The caller
|
||||
* should check the futex's value on wakeup to determine if it needs
|
||||
* to block again.
|
||||
*/
|
||||
__syscall int k_futex_wait(struct k_futex *futex, int expected, s32_t timeout);
|
||||
|
||||
/**
|
||||
* @brief Wake one/all threads pending on a futex
|
||||
*
|
||||
* Wake up the highest priority thread pending on the supplied futex, or
|
||||
* wakeup all the threads pending on the supplied futex, and the behavior
|
||||
* depends on wake_all.
|
||||
*
|
||||
* @param futex Futex to wake up pending threads.
|
||||
* @param wake_all If true, wake up all pending threads; If false,
|
||||
* wakeup the highest priority thread.
|
||||
* @retval -EACCES Caller does not have access to the futex address.
|
||||
* @retval -EINVAL Futex parameter address not recognized by the kernel.
|
||||
* @retval Number of threads that were woken up.
|
||||
*/
|
||||
__syscall int k_futex_wake(struct k_futex *futex, bool wake_all);
|
||||
|
||||
/** @} */
|
||||
#endif
|
||||
|
||||
struct k_fifo {
|
||||
struct k_queue _queue;
|
||||
};
|
||||
|
|
|
@ -46,6 +46,7 @@ target_sources_if_kconfig( kernel PRIVATE poll.c)
|
|||
target_sources_ifdef(
|
||||
CONFIG_USERSPACE
|
||||
kernel PRIVATE
|
||||
futex.c
|
||||
mem_domain.c
|
||||
userspace_handler.c
|
||||
userspace.c
|
||||
|
|
116
kernel/futex.c
Normal file
116
kernel/futex.c
Normal file
|
@ -0,0 +1,116 @@
|
|||
/*
|
||||
* Copyright (c) 2019 Intel corporation
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include <kernel.h>
|
||||
#include <kernel_structs.h>
|
||||
#include <spinlock.h>
|
||||
#include <kswap.h>
|
||||
#include <syscall_handler.h>
|
||||
#include <init.h>
|
||||
#include <ksched.h>
|
||||
|
||||
static struct z_futex_data *k_futex_find_data(struct k_futex *futex)
|
||||
{
|
||||
struct _k_object *obj;
|
||||
|
||||
obj = z_object_find(futex);
|
||||
if (obj == NULL || obj->type != K_OBJ_FUTEX) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return (struct z_futex_data *)obj->data;
|
||||
}
|
||||
|
||||
void z_impl_k_futex_init(struct k_futex *futex)
|
||||
{
|
||||
futex->val = 0U;
|
||||
z_object_init(futex);
|
||||
}
|
||||
|
||||
Z_SYSCALL_HANDLER(k_futex_init, futex)
|
||||
{
|
||||
if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) {
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
z_impl_k_futex_init((struct k_futex *)futex);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int z_impl_k_futex_wake(struct k_futex *futex, bool wake_all)
|
||||
{
|
||||
k_spinlock_key_t key;
|
||||
unsigned int woken = 0;
|
||||
struct k_thread *thread;
|
||||
struct z_futex_data *futex_data;
|
||||
|
||||
futex_data = k_futex_find_data(futex);
|
||||
if (futex_data == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
key = k_spin_lock(&futex_data->lock);
|
||||
|
||||
do {
|
||||
thread = z_unpend_first_thread(&futex_data->wait_q);
|
||||
if (thread) {
|
||||
z_ready_thread(thread);
|
||||
z_set_thread_return_value(thread, 0);
|
||||
woken++;
|
||||
}
|
||||
} while (thread && wake_all);
|
||||
|
||||
z_reschedule(&futex_data->lock, key);
|
||||
|
||||
return woken;
|
||||
}
|
||||
|
||||
Z_SYSCALL_HANDLER(k_futex_wake, futex, wake_all)
|
||||
{
|
||||
if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) {
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
return z_impl_k_futex_wake((struct k_futex *)futex, (bool)wake_all);
|
||||
}
|
||||
|
||||
int z_impl_k_futex_wait(struct k_futex *futex, int expected, s32_t timeout)
|
||||
{
|
||||
int ret;
|
||||
k_spinlock_key_t key;
|
||||
struct z_futex_data *futex_data;
|
||||
|
||||
futex_data = k_futex_find_data(futex);
|
||||
if (futex_data == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
key = k_spin_lock(&futex_data->lock);
|
||||
|
||||
if (atomic_get(&futex->val) != (atomic_val_t)expected) {
|
||||
k_spin_unlock(&futex_data->lock, key);
|
||||
return -EAGAIN;
|
||||
}
|
||||
|
||||
ret = z_pend_curr(&futex_data->lock,
|
||||
key, &futex_data->wait_q, timeout);
|
||||
if (ret == -EAGAIN) {
|
||||
ret = -ETIMEDOUT;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
Z_SYSCALL_HANDLER(k_futex_wait, futex, expected, timeout)
|
||||
{
|
||||
if (Z_SYSCALL_MEMORY_WRITE(futex, sizeof(struct k_futex)) != 0) {
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
return z_impl_k_futex_wait((struct k_futex *)futex,
|
||||
expected, (s32_t)timeout);
|
||||
}
|
|
@ -38,6 +38,7 @@ DW_OP_fbreg = 0x91
|
|||
STACK_TYPE = "_k_thread_stack_element"
|
||||
thread_counter = 0
|
||||
sys_mutex_counter = 0
|
||||
futex_counter = 0
|
||||
|
||||
# Global type environment. Populated by pass 1.
|
||||
type_env = {}
|
||||
|
@ -56,6 +57,7 @@ class KobjectInstance:
|
|||
def __init__(self, type_obj, addr):
|
||||
global thread_counter
|
||||
global sys_mutex_counter
|
||||
global futex_counter
|
||||
|
||||
self.addr = addr
|
||||
self.type_obj = type_obj
|
||||
|
@ -72,6 +74,9 @@ class KobjectInstance:
|
|||
elif self.type_obj.name == "sys_mutex":
|
||||
self.data = "(u32_t)(&kernel_mutexes[%d])" % sys_mutex_counter
|
||||
sys_mutex_counter += 1
|
||||
elif self.type_obj.name == "k_futex":
|
||||
self.data = "(u32_t)(&futex_data[%d])" % futex_counter
|
||||
futex_counter += 1
|
||||
else:
|
||||
self.data = 0
|
||||
|
||||
|
@ -566,3 +571,6 @@ class ElfHelper:
|
|||
|
||||
def get_sys_mutex_counter(self):
|
||||
return sys_mutex_counter
|
||||
|
||||
def get_futex_counter(self):
|
||||
return futex_counter
|
||||
|
|
|
@ -86,7 +86,8 @@ kobjects = OrderedDict ([
|
|||
("k_timer", (None, False)),
|
||||
("_k_thread_stack_element", (None, False)),
|
||||
("device", (None, False)),
|
||||
("sys_mutex", (None, True))
|
||||
("sys_mutex", (None, True)),
|
||||
("k_futex", (None, True))
|
||||
])
|
||||
|
||||
|
||||
|
@ -170,6 +171,15 @@ def write_gperf_table(fp, eh, objs, static_begin, static_end):
|
|||
fp.write(", ")
|
||||
fp.write("};\n")
|
||||
|
||||
num_futex = eh.get_futex_counter()
|
||||
if (num_futex != 0):
|
||||
fp.write("static struct z_futex_data futex_data[%d] = {\n" % num_futex)
|
||||
for i in range(num_futex):
|
||||
fp.write("Z_FUTEX_DATA_INITIALIZER(futex_data[%d])" % i)
|
||||
if (i != num_futex - 1):
|
||||
fp.write(", ")
|
||||
fp.write("};\n")
|
||||
|
||||
fp.write("%%\n")
|
||||
# Setup variables for mapping thread indexes
|
||||
syms = eh.get_symbols()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue