michaelh/zephyr
1
0
Fork 0
Code Releases Activity

lib/os: Add sys_heap, a new/simpler/faster memory allocator

Browse source 
The existing mem_pool implementation has been an endless source of
frustration.  It's had alignment bugs, it's had racy behavior.  It's
never been particularly fast.  It's outrageously complicated to
configure statically.  And while its fragmentation resistance and
overhead on small blocks is good, it's space efficiencey has always
been very poor due to the four-way buddy scheme.

This patch introduces sys_heap.  It's a more or less conventional
segregated fit allocator with power-of-two buckets.  It doesn't expose
its level structure to the user at all, simply taking an arbitrarily
aligned pointer to memory.  It stores all metadata inside the heap
region.  It allocates and frees by simple pointer and not block ID.
Static initialization is trivial, and runtime initialization is only a
few cycles to format and add one block to a list header.

It has excellent space efficiency.  Chunks can be split arbitrarily in
8 byte units.  Overhead is only four bytes per allocated chunk (eight
bytes for heaps >256kb or on 64 bit systems), plus a log2-sized array
of 2-word bucket headers.  No coarse alignment restrictions on blocks,
they can be split and merged (in units of 8 bytes) arbitrarily.

It has good fragmentation resistance.  Freed blocks are always
immediately merged with adjacent free blocks.  Allocations are
attempted from a sample of the smallest bucket that might fit, falling
back rapidly to the smallest block guaranteed to fit.  Split memory
remaining in the chunk is always returned immediately to the heap for
other allocation.

It has excellent performance with firmly bounded runtime.  All
operations are constant time (though there is a search of the smallest
bucket that has a compile-time-configurable upper bound, setting this
to extreme values results in an effectively linear search of the
list), objectively fast (about a hundred instructions) and amenable to
locked operation.  No more need for fragile lock relaxation trickery.

It also contains an extensive validation and stress test framework,
something that was sorely lacking in the previous implementation.

Note that sys_heap is not a compatible API with sys_mem_pool and
k_mem_pool.  Partial wrappers for those (now-) legacy APIs will appear
later and a deprecation strategy needs to be chosen.

Signed-off-by: Andy Ross <andrew.j.ross@intel.com>
This commit is contained in:
Andy Ross 2019-07-17 09:58:25 -07:00 • committed by Andrew Boie
commit aa4227754c
6 changed files with 893 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

156
include/sys/sys_heap.h Normal file
View file

@ -0,0 +1,156 @@
/*
* Copyright (c) 2019 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef ZEPHYR_INCLUDE_SYS_SYS_HEAP_H_
#define ZEPHYR_INCLUDE_SYS_SYS_HEAP_H_
#include <kernel.h>
/* Simple, fast heap implementation.
*
* A more or less conventional segregated fit allocator with
* power-of-two buckets.
*
* Excellent space efficiency. Chunks can be split arbitrarily in 8
* byte units. Overhead is only four bytes per allocated chunk (eight
* bytes for heaps >256kb or on 64 bit systems), plus a log2-sized
* array of 2-word bucket headers. No coarse alignment restrictions
* on blocks, they can be split and merged (in units of 8 bytes)
* arbitrarily.
*
* Simple API. Initialize at runtime with any blob of memory and not
* a macro-generated, carefully aligned static array. Allocate and
* free by user pointer and not an opaque block handle.
*
* Good fragmentation resistance. Freed blocks are always immediately
* merged with adjacent free blocks. Allocations are attempted from a
* sample of the smallest bucket that might fit, falling back rapidly
* to the smallest block guaranteed to fit. Split memory remaining in
* the chunk is always returned immediately to the heap for other
* allocation.
*
* Excellent performance with firmly bounded runtime. All operations
* are constant time (though there is a search of the smallest bucket
* that has a compile-time-configurable upper bound, setting this to
* extreme values results in an effectively linear search of the
* list), objectively fast (~hundred instructions) and and amenable to
* locked operation.
*/
/* Note: the init_mem/bytes fields are for the static initializer to
* have somewhere to put the arguments. The actual heap metadata at
* runtime lives in the heap memory itself and this struct simply
* functions as an opaque pointer. Would be good to clean this up and
* put the two values somewhere else, though it would make
* SYS_HEAP_DEFINE a little hairy to write.
*/
struct sys_heap {
struct z_heap *heap;
void *init_mem;
size_t init_bytes;
};
struct z_heap_stress_result {
u32_t total_allocs;
u32_t successful_allocs;
u32_t total_frees;
u64_t accumulated_in_use_bytes;
};
/** @brief Initialize sys_heap
*
* Initializes a sys_heap struct to manage the specified memory.
*
* @param h Heap to initialize
* @param mem Untyped pointer to unused memory
* @param bytes Size of region pointed to by @a mem
*/
void sys_heap_init(struct sys_heap *h, void *mem, size_t bytes);
/** @brief Allocate memory from a sys_heap
*
* Returns a pointer to a block of unused memory in the heap. This
* memory will not otherwise be used until it is freed with
* sys_heap_free(). If no memory can be allocated, NULL will be
* returned.
*
* @note The sys_heap implementation is not internally synchronized.
* No two sys_heap functions should operate on the same heap at the
* same time. All locking must be provided by the user.
*
* @param h Heap from which to allocate
* @param bytes Number of bytes requested
* @return Pointer to memory the caller can now use
*/
void *sys_heap_alloc(struct sys_heap *h, size_t bytes);
/** @brief Free memory into a sys_heap
*
* De-allocates a pointer to memory previously returned from
* sys_heap_alloc such that it can be used for other purposes. The
* caller must not use the memory region after entry to this function.
*
* @note The sys_heap implementation is not internally synchronized.
* No two sys_heap functions should operate on the same heap at the
* same time. All locking must be provided by the user.
*
* @param h Heap to which to return the memory
* @param mem A pointer previously returned from sys_heap_alloc()
*/
void sys_heap_free(struct sys_heap *h, void *mem);
/** @brief Validate heap integrity
*
* Validates the internal integrity of a sys_heap. Intended for unit
* test and validation code, though potentially useful as a user API
* for applications with complicated runtime reliability requirements.
* Note: this cannot catch every possible error, but if it returns
* true then the heap is in a consistent state and can correctly
* handle any sys_heap_alloc() request and free any live pointer
* returned from a previou allocation.
*
* @param h Heap to validate
* @return true, if the heap is valid, otherwise false
*/
bool sys_heap_validate(struct sys_heap *h);
/** @brief sys_heap stress test rig
*
* Test rig for heap allocation validation. This will loop for @a
* op_count cycles, in each iteration making a random choice to
* allocate or free a pointer of randomized (power law) size based on
* heuristics designed to keep the heap in a state where it is near @a
* target_percent full. Allocation and free operations are provided
* by the caller as callbacks (i.e. this can in theory test any heap).
* Results, including counts of frees and successful/unsuccessful
* allocations, are returnewd via the @result struct.
*
* @param alloc Callback to perform an allocation. Passes back the @a
* arg parameter as a context handle.
* @param free Callback to perform a free of a pointer returned from
* @a alloc. Passes back the @a arg parameter as a
* context handle.
* @param arg Context handle to pass back to the callbacks
* @param total_bytes Size of the byte array the heap was initialized in
* @param op_count How many iterations to test
* @param scratch_mem A pointer to scratch memory to be used by the
* test. Should be about 1/2 the size of the heap
* for tests that need to stress fragmentation.
* @param scratch_bytes Size of the memory pointed to by @a scratch_mem
* @param target_percent Percentage fill value (1-100) to which the
* random allocation choices will seek. High
* values will result in significant allocation
* failures and a very fragmented heap.
* @param result Struct into which to store test results.
*/
void sys_heap_stress(void *(*alloc)(void *arg, size_t bytes),
void (*free)(void *arg, void *p),
void *arg, size_t total_bytes,
u32_t op_count,
void *scratch_mem, size_t scratch_bytes,
int target_percent,
struct z_heap_stress_result *result);
#endif /* ZEPHYR_INCLUDE_SYS_SYS_HEAP_H_ */

2
lib/os/CMakeLists.txt
View file

@ -19,6 +19,8 @@ zephyr_sources(
thread_entry.c thread_entry.c
timeutil.c timeutil.c
work_q.c work_q.c
heap.c
heap-validate.c
) )
zephyr_sources_ifdef(CONFIG_JSON_LIBRARY json.c) zephyr_sources_ifdef(CONFIG_JSON_LIBRARY json.c)

25
lib/os/Kconfig
View file

@ -21,4 +21,29 @@ config BASE64
help help
Enable base64 encoding and decoding functionality Enable base64 encoding and decoding functionality
config SYS_HEAP_VALIDATE
bool "Enable internal heap validity checking"
help
The sys_heap implementation is instrumented for extensive
internal validation. Leave this off by default, unless
modifying the heap code or (maybe) when running in
environments that require sensitive detection of memory
corruption.
config SYS_HEAP_ALLOC_LOOPS
int "Number of tries in the inner heap allocation loop"
default 3
help
The sys_heap allocator bounds the number of tries from the
smallest chunk level (the one that might not fit the
requested allocation) to maintain constant time performance.
Setting this to a high level will cause the heap to return
more successful allocations in situations of high
fragmentation, at the cost of potentially significant
(linear time) searching of the free list. The default is
three, which results in an allocator with good statistical
properties ("most" allocations that fit will succeed) but
keeps the maximum runtime at a tight bound so that the heap
is useful in locked or ISR contexts.
endmenu endmenu

308
lib/os/heap-validate.c Normal file
View file

@ -0,0 +1,308 @@
/*
* Copyright (c) 2019 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <sys/sys_heap.h>
#include <kernel.h>
#include "heap.h"
/* White-box sys_heap validation code. Uses internal data structures.
* Not expected to be useful in production apps. This checks every
* header field of every chunk and returns true if the totality of the
* data structure is a valid heap. It doesn't necessarily tell you
* that it is the CORRECT heap given the history of alloc/free calls
* that it can't inspect. In a pathological case, you can imagine
* something scribbling a copy of a previously-valid heap on top of a
* running one and corrupting it. YMMV.
*/
static size_t max_chunkid(struct z_heap *h)
{
return h->len - bytes_to_chunksz(h, 1);
}
static bool in_bounds(struct z_heap *h, chunkid_t c)
{
return (c >= h->chunk0)
&& (c <= max_chunkid(h))
&& (size(h, c) < h->len);
}
static bool valid_chunk(struct z_heap *h, chunkid_t c)
{
return (size(h, c) > 0
&& (c + size(h, c) <= h->len)
&& in_bounds(h, c)
&& ((c == h->chunk0) || in_bounds(h, c - left_size(h, c)))
&& (used(h, c) || in_bounds(h, free_prev(h, c)))
&& (used(h, c) || in_bounds(h, free_next(h, c))));
}
/* Validate multiple state dimensions for the bucket "next" pointer
* and see that they match. Probably should unify the design a
* bit...
*/
static inline void check_nexts(struct z_heap *h, int bidx)
{
struct z_heap_bucket *b = &h->buckets[bidx];
bool emptybit = (h->avail_buckets & (1 << bidx)) == 0;
bool emptylist = b->next == 0;
bool emptycount = b->list_size == 0;
bool empties_match = emptybit == emptylist && emptybit == emptycount;
(void)empties_match;
CHECK(empties_match);
if (b->next != 0) {
CHECK(valid_chunk(h, b->next));
}
if (b->list_size == 2) {
CHECK(free_next(h, b->next) == free_prev(h, b->next));
CHECK(free_next(h, b->next) != b->next);
} else if (b->list_size == 1) {
CHECK(free_next(h, b->next) == free_prev(h, b->next));
CHECK(free_next(h, b->next) == b->next);
}
}
bool sys_heap_validate(struct sys_heap *heap)
{
struct z_heap *h = heap->heap;
chunkid_t c;
/* Check the free lists: entry count should match, empty bit
* should be correct, and all chunk entries should point into
* valid unused chunks. Mark those chunks USED, temporarily.
*/
for (int b = 0; b <= bucket_idx(h, h->len); b++) {
chunkid_t c0 = h->buckets[b].next;
u32_t n = 0;
check_nexts(h, b);
for (c = c0; c != 0 && (n == 0 || c != c0);
n++, c = free_next(h, c)) {
if (!valid_chunk(h, c)) {
return false;
}
chunk_set_used(h, c, true);
}
bool empty = (h->avail_buckets & (1 << b)) == 0;
bool zero = n == 0;
if (empty != zero) {
return false;
}
if (empty && h->buckets[b].next != 0) {
return false;
}
if (n != h->buckets[b].list_size) {
return false;
}
}
/* Walk through the chunks linearly, verifying sizes and end
* pointer and that the all chunks are now USED (i.e. all free
* blocks were found during enumeration). Mark all blocks
* UNUSED
*/
size_t prev_size = 0;
for (c = h->chunk0; c <= max_chunkid(h); c = right_chunk(h, c)) {
if (!valid_chunk(h, c)) {
return false;
}
if (!used(h, c)) {
return false;
}
if (c != h->chunk0) {
if (left_size(h, c) != prev_size) {
return false;
}
}
prev_size = size(h, c);
chunk_set_used(h, c, false);
}
if (c != h->len) {
return false; /* Should have exactly consumed the buffer */
}
/* Go through the free lists again checking that the linear
* pass caught all the blocks and that they now show UNUSED.
* Mark them USED.
*/
for (int b = 0; b <= bucket_idx(h, h->len); b++) {
chunkid_t c0 = h->buckets[b].next;
int n = 0;
if (c0 == 0) {
continue;
}
for (c = c0; n == 0 || c != c0; n++, c = free_next(h, c)) {
if (used(h, c)) {
return false;
}
chunk_set_used(h, c, true);
}
}
/* Now we are valid, but have managed to invert all the in-use
* fields. One more linear pass to fix them up
*/
for (c = h->chunk0; c <= max_chunkid(h); c = right_chunk(h, c)) {
chunk_set_used(h, c, !used(h, c));
}
return true;
}
struct z_heap_stress_rec {
void *(*alloc)(void *arg, size_t bytes);
void (*free)(void *arg, void *p);
void *arg;
size_t total_bytes;
struct z_heap_stress_block *blocks;
size_t nblocks;
size_t blocks_alloced;
size_t bytes_alloced;
u32_t target_percent;
};
struct z_heap_stress_block {
void *ptr;
size_t sz;
};
/* Very simple LCRNG (from https://nuclear.llnl.gov/CNP/rng/rngman/node4.html)
*
* Here to guarantee cross-platform test repeatability.
*/
static u32_t rand32(void)
{
static u64_t state = 123456789; /* seed */
state = state * 2862933555777941757UL + 3037000493UL;
return (u32_t)(state >> 32);
}
static bool rand_alloc_choice(struct z_heap_stress_rec *sr)
{
/* Edge cases: no blocks allocated, and no space for a new one */
if (sr->blocks_alloced == 0) {
return true;
} else if (sr->blocks_alloced >= sr->nblocks) {
return false;
}
/* The way this works is to scale the chance of choosing to
* allocate vs. free such that it's even odds when the heap is
* at the target percent, with linear tapering on the low
* slope (i.e. we choose to always allocate with an empty
* heap, allocate 50% of the time when the heap is exactly at
* the target, and always free when above the target). In
* practice, the operations aren't quite symmetric (you can
* always free, but your allocation might fail), and the units
* aren't matched (we're doing math based on bytes allocated
* and ignoring the overhead) but this is close enough. And
* yes, the math here is coarse (in units of percent), but
* that's good enough and fits well inside 32 bit quantities.
* (Note precision issue when heap size is above 40MB
* though!).
*/
__ASSERT(sr->total_bytes < 0xffffffffU / 100, "too big for u32!");
u32_t full_pct = (100 * sr->bytes_alloced) / sr->total_bytes;
u32_t target = sr->target_percent ? sr->target_percent : 1;
u32_t free_chance = 0xffffffffU;
if (full_pct < sr->target_percent) {
free_chance = full_pct * (0x80000000U / target);
}
return rand32() > free_chance;
}
/* Chooses a size of block to allocate, logarithmically favoring
* smaller blocks (i.e. blocks twice as large are half as frequent
*/
static size_t rand_alloc_size(struct z_heap_stress_rec *sr)
{
ARG_UNUSED(sr);
/* Min scale of 4 means that the half of the requests in the
* smallest size have an average size of 8
*/
int scale = 4 + __builtin_clz(rand32());
return rand32() & ((1 << scale) - 1);
}
/* Returns the index of a randomly chosen block to free */
static size_t rand_free_choice(struct z_heap_stress_rec *sr)
{
return rand32() % sr->blocks_alloced;
}
/* General purpose heap stress test. Takes function pointers to allow
* for testing multiple heap APIs with the same rig. The alloc and
* free functions are passed back the argument as a context pointer.
* The "log" function is for readable user output. The total_bytes
* argument should reflect the size of the heap being tested. The
* scratch array is used to store temporary state and should be sized
* about half as large as the heap itself. Returns true on success.
*/
void sys_heap_stress(void *(*alloc)(void *arg, size_t bytes),
void (*free)(void *arg, void *p),
void *arg, size_t total_bytes,
u32_t op_count,
void *scratch_mem, size_t scratch_bytes,
int target_percent,
struct z_heap_stress_result *result)
{
struct z_heap_stress_rec sr = {
.alloc = alloc,
.free = free,
.arg = arg,
.total_bytes = total_bytes,
.blocks = scratch_mem,
.nblocks = scratch_bytes / sizeof(struct z_heap_stress_block),
.target_percent = target_percent,
};
*result = (struct z_heap_stress_result) {0};
for (u32_t i = 0; i < op_count; i++) {
if (rand_alloc_choice(&sr)) {
size_t sz = rand_alloc_size(&sr);
void *p = sr.alloc(sr.arg, sz);
result->total_allocs++;
if (p != NULL) {
result->successful_allocs++;
sr.blocks[sr.blocks_alloced].ptr = p;
sr.blocks[sr.blocks_alloced].sz = sz;
sr.blocks_alloced++;
sr.bytes_alloced += sz;
}
} else {
int b = rand_free_choice(&sr);
void *p = sr.blocks[b].ptr;
size_t sz = sr.blocks[b].sz;
result->total_frees++;
sr.blocks[b] = sr.blocks[sr.blocks_alloced - 1];
sr.blocks_alloced--;
sr.bytes_alloced -= sz;
sr.free(sr.arg, p);
}
result->accumulated_in_use_bytes += sr.bytes_alloced;
}
}

238
lib/os/heap.c Normal file
View file

@ -0,0 +1,238 @@
/*
* Copyright (c) 2019 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <sys/sys_heap.h>
#include <kernel.h>
#include "heap.h"
static void *chunk_mem(struct z_heap *h, chunkid_t c)
{
u8_t *ret = ((u8_t *)&h->buf[c]) + chunk_header_bytes(h);
CHECK(!(((size_t)ret) & (big_heap(h) ? 7 : 3)));
return ret;
}
static void free_list_remove(struct z_heap *h, int bidx,
chunkid_t c)
{
struct z_heap_bucket *b = &h->buckets[bidx];
CHECK(!used(h, c));
CHECK(b->next != 0);
CHECK(b->list_size > 0);
CHECK((((h->avail_buckets & (1 << bidx)) == 0)
== (h->buckets[bidx].next == 0)));
b->list_size--;
if (b->list_size == 0) {
h->avail_buckets &= ~(1 << bidx);
b->next = 0;
} else {
chunkid_t first = free_prev(h, c), second = free_next(h, c);
b->next = second;
chunk_set(h, first, FREE_NEXT, second);
chunk_set(h, second, FREE_PREV, first);
}
}
static void free_list_add(struct z_heap *h, chunkid_t c)
{
int b = bucket_idx(h, size(h, c));
if (h->buckets[b].list_size++ == 0) {
CHECK(h->buckets[b].next == 0);
CHECK((h->avail_buckets & (1 << b)) == 0);
/* Empty list, first item */
h->avail_buckets |= (1 << b);
h->buckets[b].next = c;
chunk_set(h, c, FREE_PREV, c);
chunk_set(h, c, FREE_NEXT, c);
} else {
/* Insert before (!) the "next" pointer */
chunkid_t second = h->buckets[b].next;
chunkid_t first = free_prev(h, second);
chunk_set(h, c, FREE_PREV, first);
chunk_set(h, c, FREE_NEXT, second);
chunk_set(h, first, FREE_NEXT, c);
chunk_set(h, second, FREE_PREV, c);
}
CHECK(h->avail_buckets & (1 << bucket_idx(h, size(h, c))));
}
static ALWAYS_INLINE bool last_chunk(struct z_heap *h, chunkid_t c)
{
return (c + size(h, c)) == h->len;
}
/* Allocates (fit check has already been perfomred) from the next
* chunk at the specified bucket level
*/
static void *split_alloc(struct z_heap *h, int bidx, size_t sz)
{
CHECK(h->buckets[bidx].next != 0
&& sz <= size(h, h->buckets[bidx].next));
chunkid_t c = h->buckets[bidx].next;
free_list_remove(h, bidx, c);
/* Split off remainder if it's usefully large */
size_t rem = size(h, c) - sz;
CHECK(rem < h->len);
if (rem >= (big_heap(h) ? 2 : 1)) {
chunkid_t c2 = c + sz;
chunkid_t c3 = right_chunk(h, c);
chunk_set(h, c, SIZE_AND_USED, sz);
chunk_set(h, c2, SIZE_AND_USED, rem);
chunk_set(h, c2, LEFT_SIZE, sz);
if (!last_chunk(h, c2)) {
chunk_set(h, c3, LEFT_SIZE, rem);
}
free_list_add(h, c2);
}
chunk_set_used(h, c, true);
return chunk_mem(h, c);
}
void sys_heap_free(struct sys_heap *heap, void *mem)
{
if (mem == NULL) {
return; /* ISO C free() semantics */
}
struct z_heap *h = heap->heap;
chunkid_t c = ((u8_t *)mem - chunk_header_bytes(h)
- (u8_t *)h->buf) / CHUNK_UNIT;
/* Merge with right chunk? We can just absorb it. */
if (!last_chunk(h, c) && !used(h, right_chunk(h, c))) {
chunkid_t rc = right_chunk(h, c);
size_t newsz = size(h, c) + size(h, rc);
free_list_remove(h, bucket_idx(h, size(h, rc)), rc);
chunk_set(h, c, SIZE_AND_USED, newsz);
if (!last_chunk(h, c)) {
chunk_set(h, right_chunk(h, c), LEFT_SIZE, newsz);
}
}
/* Merge with left chunk? It absorbs us. */
if (c != h->chunk0 && !used(h, left_chunk(h, c))) {
chunkid_t lc = left_chunk(h, c);
chunkid_t rc = right_chunk(h, c);
size_t csz = size(h, c);
size_t merged_sz = csz + size(h, lc);
free_list_remove(h, bucket_idx(h, size(h, lc)), lc);
chunk_set(h, lc, SIZE_AND_USED, merged_sz);
if (!last_chunk(h, lc)) {
chunk_set(h, rc, LEFT_SIZE, merged_sz);
}
c = lc;
}
chunk_set_used(h, c, false);
free_list_add(h, c);
}
void *sys_heap_alloc(struct sys_heap *heap, size_t bytes)
{
struct z_heap *h = heap->heap;
size_t sz = bytes_to_chunksz(h, bytes);
int bi = bucket_idx(h, sz);
struct z_heap_bucket *b = &h->buckets[bi];
if (bytes == 0 || bi > bucket_idx(h, h->len)) {
return NULL;
}
/* First try a bounded count of items from the minimal bucket
* size. These may not fit, trying (e.g.) three means that
* (assuming that chunk sizes are evenly distributed[1]) we
* have a 7/8 chance of finding a match, thus keeping the
* number of such blocks consumed by allocation higher than
* the number of smaller blocks created by fragmenting larger
* ones.
*
* [1] In practice, they are never evenly distributed, of
* course. But even in pathological situations we still
* maintain our constant time performance and at worst see
* fragmentation waste of the order of the block allocated
* only.
*/
int loops = MIN(b->list_size, CONFIG_SYS_HEAP_ALLOC_LOOPS);
for (int i = 0; i < loops; i++) {
CHECK(b->next != 0);
if (size(h, b->next) >= sz) {
return split_alloc(h, bi, sz);
}
b->next = free_next(h, b->next);
}
/* Otherwise pick the smallest non-empty bucket guaranteed to
* fit and use that unconditionally.
*/
size_t bmask = h->avail_buckets & ~((1 << (bi + 1)) - 1);
if ((bmask & h->avail_buckets) != 0) {
int minbucket = __builtin_ctz(bmask & h->avail_buckets);
return split_alloc(h, minbucket, sz);
}
return NULL;
}
void sys_heap_init(struct sys_heap *heap, void *mem, size_t bytes)
{
/* Must fit in a 32 bit count of u64's */
#if __SIZEOF_SIZE_T__ > 4
CHECK(bytes < 0x800000000ULL);
#endif
/* Round the start up, the end down */
size_t addr = ((size_t)mem + CHUNK_UNIT - 1) & ~(CHUNK_UNIT - 1);
size_t end = ((size_t)mem + bytes) & ~(CHUNK_UNIT - 1);
size_t buf_sz = (end - addr) / CHUNK_UNIT;
size_t hdr_chunks = chunksz(sizeof(struct z_heap));
CHECK(end > addr);
struct z_heap *h = (struct z_heap *)addr;
heap->heap = (struct z_heap *)addr;
h->buf = (u64_t *)addr;
h->buckets = (void *)(addr + CHUNK_UNIT * hdr_chunks);
h->len = buf_sz;
h->size_mask = (1 << (big_heap(h) ? 31 : 15)) - 1;
h->avail_buckets = 0;
size_t buckets_bytes = ((bucket_idx(h, buf_sz) + 1)
* sizeof(struct z_heap_bucket));
h->chunk0 = hdr_chunks + chunksz(buckets_bytes);
for (int i = 0; i <= bucket_idx(heap->heap, heap->heap->len); i++) {
heap->heap->buckets[i].list_size = 0;
heap->heap->buckets[i].next = 0;
}
chunk_set(h, h->chunk0, SIZE_AND_USED, buf_sz - h->chunk0);
free_list_add(h, h->chunk0);
}

164
lib/os/heap.h Normal file
View file

@ -0,0 +1,164 @@
/*
* Copyright (c) 2019 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef ZEPHYR_INCLUDE_LIB_OS_HEAP_H_
#define ZEPHYR_INCLUDE_LIB_OS_HEAP_H_
/*
* Internal heap APIs
*/
/* Theese validation checks are non-trivially expensive, so enable
* only when debugging the heap code. They shouldn't be routine
* assertions.
*/
#ifdef CONFIG_SYS_HEAP_VALIDATE
#define CHECK(x) __ASSERT(x, "")
#else
#define CHECK(x) /**/
#endif
/* Chunks are identified by their offset in 8 byte units from the
* first address in the buffer (a zero-valued chunkid_t is used as a
* null; that chunk would always point into the metadata at the start
* of the heap and cannot be allocated). They are prefixed by a
* variable size header that depends on the size of the heap. Heaps
* with fewer than 2^15 units (256kb) of storage use shorts to store
* the fields, otherwise the units are 32 bit integers for a 16Gb heap
* space (larger spaces really aren't in scope for this code, but
* could be handled similarly I suppose). Because of that design
* there's a certain amount of boilerplate API needed to expose the
* field accessors since we can't use natural syntax.
*
* The fields are:
* SIZE_AND_USED: the total size (including header) of the chunk in
* 8-byte units. The top bit stores a "used" flag.
* LEFT_SIZE: The size of the left (next lower chunk in memory)
* neighbor chunk.
* FREE_PREV: Chunk ID of the previous node in a free list.
* FREE_NEXT: Chunk ID of the next node in a free list.
*
* The free lists are circular lists, one for each power-of-two size
* category. The free list pointers exist only for free chunks,
* obviously. This memory is part of the user's buffer when
* allocated.
*/
typedef size_t chunkid_t;
#define CHUNK_UNIT 8
enum chunk_fields { SIZE_AND_USED, LEFT_SIZE, FREE_PREV, FREE_NEXT };
struct z_heap {
u64_t *buf;
struct z_heap_bucket *buckets;
u32_t len;
u32_t size_mask;
u32_t chunk0;
u32_t avail_buckets;
};
struct z_heap_bucket {
chunkid_t next;
size_t list_size;
};
static inline bool big_heap(struct z_heap *h)
{
return sizeof(size_t) > 4 || h->len > 0x7fff;
}
static inline size_t chunk_field(struct z_heap *h, chunkid_t c,
enum chunk_fields f)
{
void *cmem = &h->buf[c];
if (big_heap(h)) {
return ((u32_t *)cmem)[f];
} else {
return ((u16_t *)cmem)[f];
}
}
static inline void chunk_set(struct z_heap *h, chunkid_t c,
enum chunk_fields f, chunkid_t val)
{
CHECK(c >= h->chunk0 && c < h->len);
CHECK((val & ~((h->size_mask << 1) + 1)) == 0);
CHECK((val & h->size_mask) < h->len);
void *cmem = &h->buf[c];
if (big_heap(h)) {
((u32_t *)cmem)[f] = (u32_t) val;
} else {
((u16_t *)cmem)[f] = (u16_t) val;
}
}
static inline chunkid_t used(struct z_heap *h, chunkid_t c)
{
return (chunk_field(h, c, SIZE_AND_USED) & ~h->size_mask) != 0;
}
static ALWAYS_INLINE chunkid_t size(struct z_heap *h, chunkid_t c)
{
return chunk_field(h, c, SIZE_AND_USED) & h->size_mask;
}
static inline void chunk_set_used(struct z_heap *h, chunkid_t c,
bool used)
{
chunk_set(h, c, SIZE_AND_USED,
size(h, c) | (used ? (h->size_mask + 1) : 0));
}
static inline chunkid_t left_size(struct z_heap *h, chunkid_t c)
{
return chunk_field(h, c, LEFT_SIZE);
}
static inline chunkid_t free_prev(struct z_heap *h, chunkid_t c)
{
return chunk_field(h, c, FREE_PREV);
}
static inline chunkid_t free_next(struct z_heap *h, chunkid_t c)
{
return chunk_field(h, c, FREE_NEXT);
}
static inline chunkid_t left_chunk(struct z_heap *h, chunkid_t c)
{
return c - left_size(h, c);
}
static inline chunkid_t right_chunk(struct z_heap *h, chunkid_t c)
{
return c + size(h, c);
}
static inline size_t chunk_header_bytes(struct z_heap *h)
{
return big_heap(h) ? 8 : 4;
}
static inline size_t chunksz(size_t bytes)
{
return (bytes + CHUNK_UNIT - 1) / CHUNK_UNIT;
}
static inline size_t bytes_to_chunksz(struct z_heap *h, size_t bytes)
{
return chunksz(chunk_header_bytes(h) + bytes);
}
static int bucket_idx(struct z_heap *h, size_t sz)
{
/* A chunk of size 2 is the minimum size on big heaps */
return 31 - __builtin_clz(sz) - (big_heap(h) ? 1 : 0);
}
#endif /* ZEPHYR_INCLUDE_LIB_OS_HEAP_H_ */