2016-06-17 13:09:40 -07:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2016 Intel Corporation
|
|
|
|
*
|
2017-01-18 17:01:01 -08:00
|
|
|
* SPDX-License-Identifier: Apache-2.0
|
2016-06-17 13:09:40 -07:00
|
|
|
*/
|
|
|
|
|
2016-11-08 10:36:50 -05:00
|
|
|
#include <kernel_structs.h>
|
|
|
|
#include <offsets_short.h>
|
2016-06-17 13:09:40 -07:00
|
|
|
|
|
|
|
/* exports */
|
2017-04-06 15:30:27 -07:00
|
|
|
GTEXT(__swap)
|
2019-03-14 09:20:46 -06:00
|
|
|
GTEXT(z_thread_entry_wrapper)
|
2016-06-17 13:09:40 -07:00
|
|
|
|
2016-06-29 16:29:15 -07:00
|
|
|
/* imports */
|
2019-09-19 09:25:19 +02:00
|
|
|
GTEXT(sys_trace_thread_switched_in)
|
2016-10-25 11:47:52 -07:00
|
|
|
GTEXT(_k_neg_eagain)
|
2016-06-29 16:29:15 -07:00
|
|
|
|
2017-04-06 15:30:27 -07:00
|
|
|
/* unsigned int __swap(unsigned int key)
|
2016-06-17 13:09:40 -07:00
|
|
|
*
|
|
|
|
* Always called with interrupts locked
|
|
|
|
*/
|
2017-04-06 15:30:27 -07:00
|
|
|
SECTION_FUNC(exception.other, __swap)
|
2016-06-17 13:09:40 -07:00
|
|
|
|
2018-07-23 13:57:06 +05:30
|
|
|
#ifdef CONFIG_EXECUTION_BENCHMARKING
|
|
|
|
/* Get a reference to _kernel in r10 */
|
|
|
|
movhi r10, %hi(_kernel)
|
|
|
|
ori r10, r10, %lo(_kernel)
|
|
|
|
|
|
|
|
/* Get the pointer to kernel->current */
|
|
|
|
ldw r11, _kernel_offset_to_current(r10)
|
|
|
|
stw r2, _thread_offset_to_r16(r11)
|
|
|
|
stw r3, _thread_offset_to_r17(r11)
|
|
|
|
stw r4, _thread_offset_to_r18(r11)
|
|
|
|
stw ra, _thread_offset_to_ra(r11)
|
|
|
|
stw sp, _thread_offset_to_sp(r11)
|
|
|
|
|
|
|
|
call read_timer_start_of_swap
|
|
|
|
/* Get a reference to _kernel in r10 */
|
|
|
|
movhi r10, %hi(_kernel)
|
|
|
|
ori r10, r10, %lo(_kernel)
|
|
|
|
|
|
|
|
/* Get the pointer to kernel->current */
|
|
|
|
ldw r11, _kernel_offset_to_current(r10)
|
|
|
|
ldw r2, _thread_offset_to_r16(r11)
|
|
|
|
ldw r3, _thread_offset_to_r17(r11)
|
|
|
|
ldw r4, _thread_offset_to_r18(r11)
|
|
|
|
ldw ra, _thread_offset_to_ra(r11)
|
|
|
|
ldw sp, _thread_offset_to_sp(r11)
|
|
|
|
|
|
|
|
#endif
|
2016-11-08 10:36:50 -05:00
|
|
|
/* Get a reference to _kernel in r10 */
|
|
|
|
movhi r10, %hi(_kernel)
|
|
|
|
ori r10, r10, %lo(_kernel)
|
2016-06-17 13:09:40 -07:00
|
|
|
|
2016-12-23 08:35:34 -05:00
|
|
|
/* Get the pointer to kernel->current */
|
2016-11-08 10:36:50 -05:00
|
|
|
ldw r11, _kernel_offset_to_current(r10)
|
2016-06-17 13:09:40 -07:00
|
|
|
|
|
|
|
/* Store all the callee saved registers. We either got here via
|
2017-04-06 15:30:27 -07:00
|
|
|
* an exception or from a cooperative invocation of __swap() from C
|
2016-06-17 13:09:40 -07:00
|
|
|
* domain, so all the caller-saved registers have already been
|
|
|
|
* saved by the exception asm or the calling C code already.
|
|
|
|
*/
|
2016-11-08 10:36:50 -05:00
|
|
|
stw r16, _thread_offset_to_r16(r11)
|
|
|
|
stw r17, _thread_offset_to_r17(r11)
|
|
|
|
stw r18, _thread_offset_to_r18(r11)
|
|
|
|
stw r19, _thread_offset_to_r19(r11)
|
|
|
|
stw r20, _thread_offset_to_r20(r11)
|
|
|
|
stw r21, _thread_offset_to_r21(r11)
|
|
|
|
stw r22, _thread_offset_to_r22(r11)
|
|
|
|
stw r23, _thread_offset_to_r23(r11)
|
|
|
|
stw r28, _thread_offset_to_r28(r11)
|
|
|
|
stw ra, _thread_offset_to_ra(r11)
|
|
|
|
stw sp, _thread_offset_to_sp(r11)
|
2016-06-17 13:09:40 -07:00
|
|
|
|
|
|
|
/* r4 has the 'key' argument which is the result of irq_lock()
|
|
|
|
* before this was called
|
|
|
|
*/
|
2016-11-08 10:36:50 -05:00
|
|
|
stw r4, _thread_offset_to_key(r11)
|
2016-06-17 13:09:40 -07:00
|
|
|
|
2016-10-25 11:47:52 -07:00
|
|
|
/* Populate default return value */
|
|
|
|
movhi r5, %hi(_k_neg_eagain)
|
|
|
|
ori r5, r5, %lo(_k_neg_eagain)
|
|
|
|
ldw r4, (r5)
|
2016-11-08 10:36:50 -05:00
|
|
|
stw r4, _thread_offset_to_retval(r11)
|
2016-10-25 11:47:52 -07:00
|
|
|
|
2018-04-06 07:48:53 -04:00
|
|
|
#if CONFIG_TRACING
|
2019-09-19 09:25:19 +02:00
|
|
|
call sys_trace_thread_switched_in
|
2017-05-11 13:29:15 -07:00
|
|
|
/* restore caller-saved r10 */
|
2016-11-08 10:36:50 -05:00
|
|
|
movhi r10, %hi(_kernel)
|
|
|
|
ori r10, r10, %lo(_kernel)
|
2017-05-11 13:29:15 -07:00
|
|
|
#endif
|
kernel/arch: enhance the "ready thread" cache
The way the ready thread cache was implemented caused it to not always
be "hot", i.e. there could be some misses, which happened when the
cached thread was taken out of the ready queue. When that happened, it
was not replaced immediately, since doing so could mean that the
replacement might not run because the flow could be interrupted and
another thread could take its place. This was the more conservative
approach that insured that moving a thread to the cache would never be
wasted.
However, this caused two problems:
1. The cache could not be refilled until another thread context-switched
in, since there was no thread in the cache to compare priorities
against.
2. Interrupt exit code would always have to call into C to find what
thread to run when the current thread was not coop and did not have the
scheduler locked. Furthermore, it was possible for this code path to
encounter a cold cache and then it had to find out what thread to run
the long way.
To fix this, filling the cache is now more aggressive, i.e. the next
thread to put in the cache is found even in the case the current cached
thread is context-switched out. This ensures the interrupt exit code is
much faster on the slow path. In addition, since finding the next thread
to run is now always "get it from the cache", which is a simple fetch
from memory (_kernel.ready_q.cache), there is no need to call the more
complex C code.
On the ARM FRDM K64F board, this improvement is seen:
Before:
1- Measure time to switch from ISR back to interrupted task
switching time is 215 tcs = 1791 nsec
2- Measure time from ISR to executing a different task (rescheduled)
switch time is 315 tcs = 2625 nsec
After:
1- Measure time to switch from ISR back to interrupted task
switching time is 130 tcs = 1083 nsec
2- Measure time from ISR to executing a different task (rescheduled)
switch time is 225 tcs = 1875 nsec
These are the most dramatic improvements, but most of the numbers
generated by the latency_measure test are improved.
Fixes ZEP-1401.
Change-Id: I2eaac147048b1ec71a93bd0a285e743a39533973
Signed-off-by: Benjamin Walsh <benjamin.walsh@windriver.com>
2016-12-02 10:37:27 -05:00
|
|
|
|
|
|
|
/* get cached thread to run */
|
|
|
|
ldw r2, _kernel_offset_to_ready_q_cache(r10)
|
2016-10-25 11:47:52 -07:00
|
|
|
|
|
|
|
/* At this point r2 points to the next thread to be swapped in */
|
2016-06-17 13:09:40 -07:00
|
|
|
|
kernel/arch: enhance the "ready thread" cache
The way the ready thread cache was implemented caused it to not always
be "hot", i.e. there could be some misses, which happened when the
cached thread was taken out of the ready queue. When that happened, it
was not replaced immediately, since doing so could mean that the
replacement might not run because the flow could be interrupted and
another thread could take its place. This was the more conservative
approach that insured that moving a thread to the cache would never be
wasted.
However, this caused two problems:
1. The cache could not be refilled until another thread context-switched
in, since there was no thread in the cache to compare priorities
against.
2. Interrupt exit code would always have to call into C to find what
thread to run when the current thread was not coop and did not have the
scheduler locked. Furthermore, it was possible for this code path to
encounter a cold cache and then it had to find out what thread to run
the long way.
To fix this, filling the cache is now more aggressive, i.e. the next
thread to put in the cache is found even in the case the current cached
thread is context-switched out. This ensures the interrupt exit code is
much faster on the slow path. In addition, since finding the next thread
to run is now always "get it from the cache", which is a simple fetch
from memory (_kernel.ready_q.cache), there is no need to call the more
complex C code.
On the ARM FRDM K64F board, this improvement is seen:
Before:
1- Measure time to switch from ISR back to interrupted task
switching time is 215 tcs = 1791 nsec
2- Measure time from ISR to executing a different task (rescheduled)
switch time is 315 tcs = 2625 nsec
After:
1- Measure time to switch from ISR back to interrupted task
switching time is 130 tcs = 1083 nsec
2- Measure time from ISR to executing a different task (rescheduled)
switch time is 225 tcs = 1875 nsec
These are the most dramatic improvements, but most of the numbers
generated by the latency_measure test are improved.
Fixes ZEP-1401.
Change-Id: I2eaac147048b1ec71a93bd0a285e743a39533973
Signed-off-by: Benjamin Walsh <benjamin.walsh@windriver.com>
2016-12-02 10:37:27 -05:00
|
|
|
/* the thread to be swapped in is now the current thread */
|
|
|
|
stw r2, _kernel_offset_to_current(r10)
|
|
|
|
|
2016-06-17 13:09:40 -07:00
|
|
|
/* Restore callee-saved registers and switch to the incoming
|
|
|
|
* thread's stack
|
|
|
|
*/
|
2016-11-08 10:36:50 -05:00
|
|
|
ldw r16, _thread_offset_to_r16(r2)
|
|
|
|
ldw r17, _thread_offset_to_r17(r2)
|
|
|
|
ldw r18, _thread_offset_to_r18(r2)
|
|
|
|
ldw r19, _thread_offset_to_r19(r2)
|
|
|
|
ldw r20, _thread_offset_to_r20(r2)
|
|
|
|
ldw r21, _thread_offset_to_r21(r2)
|
|
|
|
ldw r22, _thread_offset_to_r22(r2)
|
|
|
|
ldw r23, _thread_offset_to_r23(r2)
|
|
|
|
ldw r28, _thread_offset_to_r28(r2)
|
|
|
|
ldw ra, _thread_offset_to_ra(r2)
|
|
|
|
ldw sp, _thread_offset_to_sp(r2)
|
2016-10-25 11:47:52 -07:00
|
|
|
|
|
|
|
/* We need to irq_unlock(current->coopReg.key);
|
2017-04-06 15:30:27 -07:00
|
|
|
* key was supplied as argument to __swap(). Fetch it.
|
2016-10-25 11:47:52 -07:00
|
|
|
*/
|
2016-11-08 10:36:50 -05:00
|
|
|
ldw r3, _thread_offset_to_key(r2)
|
2016-10-25 11:47:52 -07:00
|
|
|
|
2016-12-21 11:16:01 -05:00
|
|
|
/*
|
|
|
|
* Load return value into r2 (return value register). -EAGAIN unless
|
2019-09-21 16:25:56 -07:00
|
|
|
* someone previously called z_arch_thread_return_value_set(). Do this before
|
2016-12-21 11:16:01 -05:00
|
|
|
* we potentially unlock interrupts.
|
2016-06-23 20:10:01 -07:00
|
|
|
*/
|
2016-11-08 10:36:50 -05:00
|
|
|
ldw r2, _thread_offset_to_retval(r2)
|
2016-06-23 20:10:01 -07:00
|
|
|
|
2016-10-25 11:47:52 -07:00
|
|
|
/* Now do irq_unlock(current->coopReg.key) */
|
2016-07-11 12:42:02 -07:00
|
|
|
#if (ALT_CPU_NUM_OF_SHADOW_REG_SETS > 0) || \
|
|
|
|
(defined ALT_CPU_EIC_PRESENT) || \
|
|
|
|
(defined ALT_CPU_MMU_PRESENT) || \
|
|
|
|
(defined ALT_CPU_MPU_PRESENT)
|
2016-06-23 20:10:01 -07:00
|
|
|
andi r3, r3, NIOS2_STATUS_PIE_MSK
|
2016-06-17 13:09:40 -07:00
|
|
|
beq r3, zero, no_unlock
|
|
|
|
rdctl r3, status
|
|
|
|
ori r3, r3, NIOS2_STATUS_PIE_MSK
|
|
|
|
wrctl status, r3
|
|
|
|
|
2016-08-18 09:25:00 -07:00
|
|
|
no_unlock:
|
2016-06-24 07:56:48 -07:00
|
|
|
#else
|
|
|
|
wrctl status, r3
|
|
|
|
#endif
|
2018-07-23 13:57:06 +05:30
|
|
|
|
|
|
|
#ifdef CONFIG_EXECUTION_BENCHMARKING
|
|
|
|
/* Get a reference to _kernel in r10 */
|
|
|
|
movhi r10, %hi(_kernel)
|
|
|
|
ori r10, r10, %lo(_kernel)
|
|
|
|
|
|
|
|
ldw r11, _kernel_offset_to_current(r10)
|
|
|
|
stw r2, _thread_offset_to_r16(r11)
|
|
|
|
stw r3, _thread_offset_to_r17(r11)
|
|
|
|
stw r4, _thread_offset_to_r18(r11)
|
|
|
|
stw ra, _thread_offset_to_ra(r11)
|
|
|
|
stw sp, _thread_offset_to_sp(r11)
|
|
|
|
|
|
|
|
call read_timer_end_of_swap
|
|
|
|
|
|
|
|
/* Get a reference to _kernel in r10 */
|
|
|
|
movhi r10, %hi(_kernel)
|
|
|
|
ori r10, r10, %lo(_kernel)
|
|
|
|
|
|
|
|
/* Get the pointer to kernel->current */
|
|
|
|
ldw r11, _kernel_offset_to_current(r10)
|
|
|
|
ldw r2, _thread_offset_to_r16(r11)
|
|
|
|
ldw r3, _thread_offset_to_r17(r11)
|
|
|
|
ldw r4, _thread_offset_to_r18(r11)
|
|
|
|
ldw ra, _thread_offset_to_ra(r11)
|
|
|
|
ldw sp, _thread_offset_to_sp(r11)
|
|
|
|
#endif
|
2016-06-17 13:09:40 -07:00
|
|
|
ret
|
|
|
|
|
|
|
|
|
2019-03-14 09:20:46 -06:00
|
|
|
/* void z_thread_entry_wrapper(void)
|
2016-06-17 13:09:40 -07:00
|
|
|
*/
|
2019-03-14 09:20:46 -06:00
|
|
|
SECTION_FUNC(TEXT, z_thread_entry_wrapper)
|
2016-06-17 13:09:40 -07:00
|
|
|
/* This all corresponds to struct init_stack_frame defined in
|
|
|
|
* thread.c. We need to take this stuff off the stack and put
|
2019-06-18 14:45:40 -04:00
|
|
|
* it in the appropriate registers
|
2016-06-17 13:09:40 -07:00
|
|
|
*/
|
|
|
|
|
|
|
|
/* Can't return from here, just put NULL in ra */
|
|
|
|
movi ra, 0
|
|
|
|
|
|
|
|
/* Calling convention has first 4 arguments in registers r4-r7. */
|
|
|
|
ldw r4, 0(sp)
|
|
|
|
ldw r5, 4(sp)
|
|
|
|
ldw r6, 8(sp)
|
|
|
|
ldw r7, 12(sp)
|
|
|
|
|
|
|
|
/* pop all the stuff that we just loaded into registers */
|
|
|
|
addi sp, sp, 16
|
|
|
|
|
2019-03-08 14:19:05 -07:00
|
|
|
call z_thread_entry
|
2016-06-17 13:09:40 -07:00
|
|
|
|