zephyr/include/sys/atomic.h
Andy Ross 820c94e5dd arch/xtensa: Inline atomics
The xtensa atomics layer was written with hand-coded assembly that had
to be called as functions.  That's needlessly slow, given that the low
level primitives are a two-instruction sequence.  Ideally the compiler
should see this as an inline to permit it to better optimize around
the needed barriers.

There was also a bug with the atomic_cas function, which had a loop
internally instead of returning the old value synchronously on a
failed swap.  That's benign right now because our existing spin lock
does nothing but retry it in a tight loop anyway, but it's incorrect
per spec and would have caused a contention hang with more elaborate
algorithms (for example a spinlock with backoff semantics).

Remove the old implementation and replace with a much smaller inline C
one based on just two assembly primitives.

This patch also contains a little bit of refactoring to address the
scheme has been split out into a separate header for each, and the
ATOMIC_OPERATIONS_CUSTOM kconfig has been renamed to
ATOMIC_OPERATIONS_ARCH to better capture what it means.

Signed-off-by: Andy Ross <andrew.j.ross@intel.com>
2021-03-08 11:14:27 -05:00

214 lines
5.2 KiB
C

/*
* Copyright (c) 1997-2015, Wind River Systems, Inc.
* Copyright (c) 2021 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef ZEPHYR_INCLUDE_SYS_ATOMIC_H_
#define ZEPHYR_INCLUDE_SYS_ATOMIC_H_
#include <stdbool.h>
#include <toolchain.h>
#include <stddef.h>
#include <zephyr/types.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef int atomic_t;
typedef atomic_t atomic_val_t;
typedef void *atomic_ptr_t;
/* Low-level primitives come in several styles: */
#if defined(CONFIG_ATOMIC_OPERATIONS_BUILTIN)
/* Default. See this file for the Doxygen reference: */
#include <sys/atomic_builtin.h>
#elif defined(CONFIG_ATOMIC_OPERATIONS_ARCH)
/* Some architectures need their own implementation */
# ifdef CONFIG_XTENSA
/* Not all Xtensa toolchains support GCC-style atomic intrinsics */
# include <arch/xtensa/atomic_xtensa.h>
# endif
#else
/* Generic-but-slow implementation based on kernel locking and syscalls */
#include <sys/atomic_c.h>
#endif
/* Portable higher-level utilities: */
/**
* @defgroup atomic_apis Atomic Services APIs
* @ingroup kernel_apis
* @{
*/
/**
* @brief Initialize an atomic variable.
*
* This macro can be used to initialize an atomic variable. For example,
* @code atomic_t my_var = ATOMIC_INIT(75); @endcode
*
* @param i Value to assign to atomic variable.
*/
#define ATOMIC_INIT(i) (i)
/**
* @cond INTERNAL_HIDDEN
*/
#define ATOMIC_BITS (sizeof(atomic_val_t) * 8)
#define ATOMIC_MASK(bit) (1U << ((uint32_t)(bit) & (ATOMIC_BITS - 1U)))
#define ATOMIC_ELEM(addr, bit) ((addr) + ((bit) / ATOMIC_BITS))
/**
* INTERNAL_HIDDEN @endcond
*/
/**
* @brief Define an array of atomic variables.
*
* This macro defines an array of atomic variables containing at least
* @a num_bits bits.
*
* @note
* If used from file scope, the bits of the array are initialized to zero;
* if used from within a function, the bits are left uninitialized.
*
* @param name Name of array of atomic variables.
* @param num_bits Number of bits needed.
*/
#define ATOMIC_DEFINE(name, num_bits) \
atomic_t name[1 + ((num_bits) - 1) / ATOMIC_BITS]
/**
* @brief Atomically test a bit.
*
* This routine tests whether bit number @a bit of @a target is set or not.
* The target may be a single atomic variable or an array of them.
*
* @param target Address of atomic variable or array.
* @param bit Bit number (starting from 0).
*
* @return true if the bit was set, false if it wasn't.
*/
static inline bool atomic_test_bit(const atomic_t *target, int bit)
{
atomic_val_t val = atomic_get(ATOMIC_ELEM(target, bit));
return (1 & (val >> (bit & (ATOMIC_BITS - 1)))) != 0;
}
/**
* @brief Atomically test and clear a bit.
*
* Atomically clear bit number @a bit of @a target and return its old value.
* The target may be a single atomic variable or an array of them.
*
* @param target Address of atomic variable or array.
* @param bit Bit number (starting from 0).
*
* @return true if the bit was set, false if it wasn't.
*/
static inline bool atomic_test_and_clear_bit(atomic_t *target, int bit)
{
atomic_val_t mask = ATOMIC_MASK(bit);
atomic_val_t old;
old = atomic_and(ATOMIC_ELEM(target, bit), ~mask);
return (old & mask) != 0;
}
/**
* @brief Atomically set a bit.
*
* Atomically set bit number @a bit of @a target and return its old value.
* The target may be a single atomic variable or an array of them.
*
* @param target Address of atomic variable or array.
* @param bit Bit number (starting from 0).
*
* @return true if the bit was set, false if it wasn't.
*/
static inline bool atomic_test_and_set_bit(atomic_t *target, int bit)
{
atomic_val_t mask = ATOMIC_MASK(bit);
atomic_val_t old;
old = atomic_or(ATOMIC_ELEM(target, bit), mask);
return (old & mask) != 0;
}
/**
* @brief Atomically clear a bit.
*
* Atomically clear bit number @a bit of @a target.
* The target may be a single atomic variable or an array of them.
*
* @param target Address of atomic variable or array.
* @param bit Bit number (starting from 0).
*
* @return N/A
*/
static inline void atomic_clear_bit(atomic_t *target, int bit)
{
atomic_val_t mask = ATOMIC_MASK(bit);
(void)atomic_and(ATOMIC_ELEM(target, bit), ~mask);
}
/**
* @brief Atomically set a bit.
*
* Atomically set bit number @a bit of @a target.
* The target may be a single atomic variable or an array of them.
*
* @param target Address of atomic variable or array.
* @param bit Bit number (starting from 0).
*
* @return N/A
*/
static inline void atomic_set_bit(atomic_t *target, int bit)
{
atomic_val_t mask = ATOMIC_MASK(bit);
(void)atomic_or(ATOMIC_ELEM(target, bit), mask);
}
/**
* @brief Atomically set a bit to a given value.
*
* Atomically set bit number @a bit of @a target to value @a val.
* The target may be a single atomic variable or an array of them.
*
* @param target Address of atomic variable or array.
* @param bit Bit number (starting from 0).
* @param val true for 1, false for 0.
*
* @return N/A
*/
static inline void atomic_set_bit_to(atomic_t *target, int bit, bool val)
{
atomic_val_t mask = ATOMIC_MASK(bit);
if (val) {
(void)atomic_or(ATOMIC_ELEM(target, bit), mask);
} else {
(void)atomic_and(ATOMIC_ELEM(target, bit), ~mask);
}
}
/**
* @}
*/
#ifdef __cplusplus
} /* extern "C" */
#endif
#endif /* ZEPHYR_INCLUDE_SYS_ATOMIC_H_ */