From 3c2e57c923c2329966f4797abebbdc18531b8e03 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 23 May 2024 17:27:18 -0400 Subject: [PATCH] drivers/timer/apic_tsc: use ICR as a fallback timeout event source This adds support for the local APIC in one-shot mode as the timeout event source for those cases where the CPU supports invariant TSC but no TSC deadline capability. It is presented as another timer choice. Existing Kconfig symbols were preserved to minimize board config disturbance. This hybrid approach was implemented kind of backward in the apic_timer driver but it is far cleaner to carry this here. Signed-off-by: Nicolas Pitre --- boards/intel/adl/Kconfig.defconfig | 2 + boards/intel/ehl/Kconfig.defconfig | 2 + boards/intel/rpl/Kconfig.defconfig | 2 + .../up_squared/Kconfig.defconfig | 2 + .../up_squared_pro_7000/Kconfig.defconfig | 2 + drivers/timer/CMakeLists.txt | 1 + drivers/timer/Kconfig.x86 | 26 +++--- drivers/timer/apic_tsc.c | 85 ++++++++++++++++--- soc/intel/apollo_lake/Kconfig.defconfig | 3 - 9 files changed, 100 insertions(+), 25 deletions(-) diff --git a/boards/intel/adl/Kconfig.defconfig b/boards/intel/adl/Kconfig.defconfig index 518f0c01ee2..3a6bcaf9ce0 100644 --- a/boards/intel/adl/Kconfig.defconfig +++ b/boards/intel/adl/Kconfig.defconfig @@ -16,6 +16,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/intel/ehl/Kconfig.defconfig b/boards/intel/ehl/Kconfig.defconfig index 9e7dcf48a04..8b8c89800da 100644 --- a/boards/intel/ehl/Kconfig.defconfig +++ b/boards/intel/ehl/Kconfig.defconfig @@ -27,6 +27,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/intel/rpl/Kconfig.defconfig b/boards/intel/rpl/Kconfig.defconfig index ca8a7742688..4eb5d8c3126 100644 --- a/boards/intel/rpl/Kconfig.defconfig +++ b/boards/intel/rpl/Kconfig.defconfig @@ -17,6 +17,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig b/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig index 78d041976f1..9c721cfcc32 100644 --- a/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig +++ b/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig @@ -18,6 +18,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig b/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig index 84744a7ea3a..e7c98f9eddf 100644 --- a/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig +++ b/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig @@ -19,6 +19,8 @@ if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/drivers/timer/CMakeLists.txt b/drivers/timer/CMakeLists.txt index c44acda2ae6..80f8431c4d0 100644 --- a/drivers/timer/CMakeLists.txt +++ b/drivers/timer/CMakeLists.txt @@ -6,6 +6,7 @@ zephyr_library_sources_ifdef(CONFIG_ALTERA_AVALON_TIMER altera_avalon_timer_hal. zephyr_library_sources_ifdef(CONFIG_AMBIQ_STIMER_TIMER ambiq_stimer.c) zephyr_library_sources_ifdef(CONFIG_APIC_TIMER apic_timer.c) zephyr_library_sources_ifdef(CONFIG_APIC_TSC_DEADLINE_TIMER apic_tsc.c) +zephyr_library_sources_ifdef(CONFIG_APIC_TIMER_TSC apic_tsc.c) zephyr_library_sources_ifdef(CONFIG_ARCV2_TIMER arcv2_timer0.c) zephyr_library_sources_ifdef(CONFIG_ARM_ARCH_TIMER arm_arch_timer.c) zephyr_library_sources_ifdef(CONFIG_INTEL_ADSP_TIMER intel_adsp_timer.c) diff --git a/drivers/timer/Kconfig.x86 b/drivers/timer/Kconfig.x86 index ea6d4a216b7..55dbf819ec8 100644 --- a/drivers/timer/Kconfig.x86 +++ b/drivers/timer/Kconfig.x86 @@ -51,6 +51,19 @@ config APIC_TSC_DEADLINE_TIMER choice for any x86 device with invariant TSC and TSC deadline capability. +config APIC_TIMER_TSC + bool "Local APIC timer using TSC time source" + depends on !SMP + select LOAPIC + select TICKLESS_CAPABLE + select TIMER_HAS_64BIT_CYCLE_COUNTER + help + If your CPU supports invariant TSC but no TSC deadline capability + then this choice will rely on the TSC as time source and the + local APIC in one-shot mode as the timeout event source. + You must know the ratio of the TSC frequency to the local APIC + timer frequency. + endchoice if APIC_TIMER @@ -66,14 +79,7 @@ config APIC_TIMER_IRQ user-configurable and almost certainly should be managed via a different mechanism. -config APIC_TIMER_TSC - bool "Use invariant TSC for sys_clock_cycle_get_32()" - select TIMER_HAS_64BIT_CYCLE_COUNTER - help - If your CPU supports invariant TSC, and you know the ratio of the - TSC frequency to CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC (the local APIC - timer frequency), then enable this for a much faster and more - accurate sys_clock_cycle_get_32(). +endif # APIC_TIMER if APIC_TIMER_TSC @@ -87,11 +93,9 @@ config APIC_TIMER_TSC_M endif # APIC_TIMER_TSC -endif # APIC_TIMER - config APIC_TIMER_IRQ_PRIORITY int "Local APIC timer interrupt priority" - depends on APIC_TIMER || APIC_TSC_DEADLINE_TIMER + depends on APIC_TIMER || APIC_TSC_DEADLINE_TIMER || APIC_TIMER_TSC default 4 help This option specifies the interrupt priority used by the diff --git a/drivers/timer/apic_tsc.c b/drivers/timer/apic_tsc.c index f2951be965f..0c617a01ca7 100644 --- a/drivers/timer/apic_tsc.c +++ b/drivers/timer/apic_tsc.c @@ -13,6 +13,37 @@ #include #include +/* + * This driver is selected when either CONFIG_APIC_TIMER_TSC or + * CONFIG_APIC_TSC_DEADLINE_TIMER is selected. The later is preferred over + * the former when the TSC deadline comparator is available. + */ +BUILD_ASSERT((!IS_ENABLED(CONFIG_APIC_TIMER_TSC) && + IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER)) || + (!IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER) && + IS_ENABLED(CONFIG_APIC_TIMER_TSC)), + "one of CONFIG_APIC_TIMER_TSC or CONFIG_APIC_TSC_DEADLINE_TIMER must be set"); + +/* + * If the TSC deadline comparator is not supported then the ICR in one-shot + * mode is used as a fallback method to trigger the next timeout interrupt. + * Those config symbols must then be defined: + * + * CONFIG_APIC_TIMER_TSC_N= + * CONFIG_APIC_TIMER_TSC_M= + * + * These are set to indicate the ratio of the TSC frequency to the local + * APIC timer frequency. This can be found via CPUID 0x15 (n = EBX, m = EAX) + * on most CPUs. + */ +#ifdef CONFIG_APIC_TIMER_TSC +#define APIC_TIMER_TSC_M CONFIG_APIC_TIMER_TSC_M +#define APIC_TIMER_TSC_N CONFIG_APIC_TIMER_TSC_N +#else +#define APIC_TIMER_TSC_M 1 +#define APIC_TIMER_TSC_N 1 +#endif + #define IA32_TSC_DEADLINE_MSR 0x6e0 #define IA32_TSC_ADJUST_MSR 0x03b @@ -78,6 +109,22 @@ static inline void wrmsr(int32_t msr, uint64_t val) __asm__ volatile("wrmsr" :: "d"(hi), "a"(lo), "c"(msr)); } +static void set_trigger(uint64_t deadline) +{ + if (IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER)) { + wrmsr(IA32_TSC_DEADLINE_MSR, deadline); + } else { + /* use the timer ICR to trigger next interrupt */ + uint64_t curr_cycle = rdtsc(); + uint64_t delta_cycles = deadline - MIN(deadline, curr_cycle); + uint64_t icr = (delta_cycles * APIC_TIMER_TSC_M) / APIC_TIMER_TSC_N; + + /* cap icr to 32 bits, and not zero */ + icr = CLAMP(icr, 1, UINT32_MAX); + x86_write_loapic(LOAPIC_TIMER_ICR, icr); + } +} + static void isr(const void *arg) { ARG_UNUSED(arg); @@ -94,7 +141,7 @@ static void isr(const void *arg) if (!IS_ENABLED(CONFIG_TICKLESS_KERNEL)) { uint64_t next_cycle = last_cycle + CYC_PER_TICK; - wrmsr(IA32_TSC_DEADLINE_MSR, next_cycle); + set_trigger(next_cycle); } k_spin_unlock(&lock, key); @@ -132,7 +179,7 @@ void sys_clock_set_timeout(int32_t ticks, bool idle) if (next_cycle < last_cycle) { next_cycle = UINT64_MAX; } - wrmsr(IA32_TSC_DEADLINE_MSR, next_cycle); + set_trigger(next_cycle); k_spin_unlock(&lock, key); } @@ -213,28 +260,44 @@ static int sys_clock_driver_init(void) #ifdef CONFIG_ASSERT uint32_t eax, ebx, ecx, edx; - ecx = 0; /* prevent compiler warning */ - __get_cpuid(CPUID_BASIC_INFO_1, &eax, &ebx, &ecx, &edx); - __ASSERT((ecx & BIT(24)) != 0, "No TSC Deadline support"); + if (IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER)) { + ecx = 0; /* prevent compiler warning */ + __get_cpuid(CPUID_BASIC_INFO_1, &eax, &ebx, &ecx, &edx); + __ASSERT((ecx & BIT(24)) != 0, "No TSC Deadline support"); + } edx = 0; /* prevent compiler warning */ __get_cpuid(0x80000007, &eax, &ebx, &ecx, &edx); __ASSERT((edx & BIT(8)) != 0, "No Invariant TSC support"); - ebx = 0; /* prevent compiler warning */ - __get_cpuid_count(CPUID_EXTENDED_FEATURES_LVL, 0, &eax, &ebx, &ecx, &edx); - __ASSERT((ebx & BIT(1)) != 0, "No TSC_ADJUST MSR support"); + if (IS_ENABLED(CONFIG_SMP)) { + ebx = 0; /* prevent compiler warning */ + __get_cpuid_count(CPUID_EXTENDED_FEATURES_LVL, 0, &eax, &ebx, &ecx, &edx); + __ASSERT((ebx & BIT(1)) != 0, "No TSC_ADJUST MSR support"); + } #endif - clear_tsc_adjust(); + if (IS_ENABLED(CONFIG_SMP)) { + clear_tsc_adjust(); + } /* Timer interrupt number is runtime-fetched, so can't use * static IRQ_CONNECT() */ irq_connect_dynamic(timer_irq(), CONFIG_APIC_TIMER_IRQ_PRIORITY, isr, 0, 0); + if (IS_ENABLED(CONFIG_APIC_TIMER_TSC)) { + uint32_t timer_conf; + + timer_conf = x86_read_loapic(LOAPIC_TIMER_CONFIG); + timer_conf &= ~0x0f; /* clear divider bits */ + timer_conf |= 0x0b; /* divide by 1 */ + x86_write_loapic(LOAPIC_TIMER_CONFIG, timer_conf); + } + lvt_reg.val = x86_read_loapic(LOAPIC_TIMER); - lvt_reg.lvt.mode = TSC_DEADLINE; + lvt_reg.lvt.mode = IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER) ? + TSC_DEADLINE : ONE_SHOT; lvt_reg.lvt.masked = 0; x86_write_loapic(LOAPIC_TIMER, lvt_reg.val); @@ -248,7 +311,7 @@ static int sys_clock_driver_init(void) last_tick = rdtsc() / CYC_PER_TICK; last_cycle = last_tick * CYC_PER_TICK; if (!IS_ENABLED(CONFIG_TICKLESS_KERNEL)) { - wrmsr(IA32_TSC_DEADLINE_MSR, last_cycle + CYC_PER_TICK); + set_trigger(last_cycle + CYC_PER_TICK); } irq_enable(timer_irq()); diff --git a/soc/intel/apollo_lake/Kconfig.defconfig b/soc/intel/apollo_lake/Kconfig.defconfig index 7ea881ce59e..ef062600dfa 100644 --- a/soc/intel/apollo_lake/Kconfig.defconfig +++ b/soc/intel/apollo_lake/Kconfig.defconfig @@ -14,9 +14,6 @@ if APIC_TIMER config APIC_TIMER_IRQ default 24 -config APIC_TIMER_TSC - default y - endif # APIC_TIMER config X86_DYNAMIC_IRQ_STUBS