diff --git a/boards/intel/adl/Kconfig.defconfig b/boards/intel/adl/Kconfig.defconfig index 518f0c01ee2..3a6bcaf9ce0 100644 --- a/boards/intel/adl/Kconfig.defconfig +++ b/boards/intel/adl/Kconfig.defconfig @@ -16,6 +16,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/intel/ehl/Kconfig.defconfig b/boards/intel/ehl/Kconfig.defconfig index 9e7dcf48a04..8b8c89800da 100644 --- a/boards/intel/ehl/Kconfig.defconfig +++ b/boards/intel/ehl/Kconfig.defconfig @@ -27,6 +27,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/intel/rpl/Kconfig.defconfig b/boards/intel/rpl/Kconfig.defconfig index ca8a7742688..4eb5d8c3126 100644 --- a/boards/intel/rpl/Kconfig.defconfig +++ b/boards/intel/rpl/Kconfig.defconfig @@ -17,6 +17,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig b/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig index 78d041976f1..9c721cfcc32 100644 --- a/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig +++ b/boards/up-bridge-the-gap/up_squared/Kconfig.defconfig @@ -18,6 +18,8 @@ config SYS_CLOCK_HW_CYCLES_PER_SEC if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig b/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig index 84744a7ea3a..e7c98f9eddf 100644 --- a/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig +++ b/boards/up-bridge-the-gap/up_squared_pro_7000/Kconfig.defconfig @@ -19,6 +19,8 @@ if APIC_TIMER config APIC_TIMER_IRQ default 24 +endif +if APIC_TIMER_TSC config APIC_TIMER_TSC_M default 3 config APIC_TIMER_TSC_N diff --git a/drivers/timer/CMakeLists.txt b/drivers/timer/CMakeLists.txt index c44acda2ae6..80f8431c4d0 100644 --- a/drivers/timer/CMakeLists.txt +++ b/drivers/timer/CMakeLists.txt @@ -6,6 +6,7 @@ zephyr_library_sources_ifdef(CONFIG_ALTERA_AVALON_TIMER altera_avalon_timer_hal. zephyr_library_sources_ifdef(CONFIG_AMBIQ_STIMER_TIMER ambiq_stimer.c) zephyr_library_sources_ifdef(CONFIG_APIC_TIMER apic_timer.c) zephyr_library_sources_ifdef(CONFIG_APIC_TSC_DEADLINE_TIMER apic_tsc.c) +zephyr_library_sources_ifdef(CONFIG_APIC_TIMER_TSC apic_tsc.c) zephyr_library_sources_ifdef(CONFIG_ARCV2_TIMER arcv2_timer0.c) zephyr_library_sources_ifdef(CONFIG_ARM_ARCH_TIMER arm_arch_timer.c) zephyr_library_sources_ifdef(CONFIG_INTEL_ADSP_TIMER intel_adsp_timer.c) diff --git a/drivers/timer/Kconfig.x86 b/drivers/timer/Kconfig.x86 index ea6d4a216b7..55dbf819ec8 100644 --- a/drivers/timer/Kconfig.x86 +++ b/drivers/timer/Kconfig.x86 @@ -51,6 +51,19 @@ config APIC_TSC_DEADLINE_TIMER choice for any x86 device with invariant TSC and TSC deadline capability. +config APIC_TIMER_TSC + bool "Local APIC timer using TSC time source" + depends on !SMP + select LOAPIC + select TICKLESS_CAPABLE + select TIMER_HAS_64BIT_CYCLE_COUNTER + help + If your CPU supports invariant TSC but no TSC deadline capability + then this choice will rely on the TSC as time source and the + local APIC in one-shot mode as the timeout event source. + You must know the ratio of the TSC frequency to the local APIC + timer frequency. + endchoice if APIC_TIMER @@ -66,14 +79,7 @@ config APIC_TIMER_IRQ user-configurable and almost certainly should be managed via a different mechanism. -config APIC_TIMER_TSC - bool "Use invariant TSC for sys_clock_cycle_get_32()" - select TIMER_HAS_64BIT_CYCLE_COUNTER - help - If your CPU supports invariant TSC, and you know the ratio of the - TSC frequency to CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC (the local APIC - timer frequency), then enable this for a much faster and more - accurate sys_clock_cycle_get_32(). +endif # APIC_TIMER if APIC_TIMER_TSC @@ -87,11 +93,9 @@ config APIC_TIMER_TSC_M endif # APIC_TIMER_TSC -endif # APIC_TIMER - config APIC_TIMER_IRQ_PRIORITY int "Local APIC timer interrupt priority" - depends on APIC_TIMER || APIC_TSC_DEADLINE_TIMER + depends on APIC_TIMER || APIC_TSC_DEADLINE_TIMER || APIC_TIMER_TSC default 4 help This option specifies the interrupt priority used by the diff --git a/drivers/timer/apic_tsc.c b/drivers/timer/apic_tsc.c index f2951be965f..0c617a01ca7 100644 --- a/drivers/timer/apic_tsc.c +++ b/drivers/timer/apic_tsc.c @@ -13,6 +13,37 @@ #include #include +/* + * This driver is selected when either CONFIG_APIC_TIMER_TSC or + * CONFIG_APIC_TSC_DEADLINE_TIMER is selected. The later is preferred over + * the former when the TSC deadline comparator is available. + */ +BUILD_ASSERT((!IS_ENABLED(CONFIG_APIC_TIMER_TSC) && + IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER)) || + (!IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER) && + IS_ENABLED(CONFIG_APIC_TIMER_TSC)), + "one of CONFIG_APIC_TIMER_TSC or CONFIG_APIC_TSC_DEADLINE_TIMER must be set"); + +/* + * If the TSC deadline comparator is not supported then the ICR in one-shot + * mode is used as a fallback method to trigger the next timeout interrupt. + * Those config symbols must then be defined: + * + * CONFIG_APIC_TIMER_TSC_N= + * CONFIG_APIC_TIMER_TSC_M= + * + * These are set to indicate the ratio of the TSC frequency to the local + * APIC timer frequency. This can be found via CPUID 0x15 (n = EBX, m = EAX) + * on most CPUs. + */ +#ifdef CONFIG_APIC_TIMER_TSC +#define APIC_TIMER_TSC_M CONFIG_APIC_TIMER_TSC_M +#define APIC_TIMER_TSC_N CONFIG_APIC_TIMER_TSC_N +#else +#define APIC_TIMER_TSC_M 1 +#define APIC_TIMER_TSC_N 1 +#endif + #define IA32_TSC_DEADLINE_MSR 0x6e0 #define IA32_TSC_ADJUST_MSR 0x03b @@ -78,6 +109,22 @@ static inline void wrmsr(int32_t msr, uint64_t val) __asm__ volatile("wrmsr" :: "d"(hi), "a"(lo), "c"(msr)); } +static void set_trigger(uint64_t deadline) +{ + if (IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER)) { + wrmsr(IA32_TSC_DEADLINE_MSR, deadline); + } else { + /* use the timer ICR to trigger next interrupt */ + uint64_t curr_cycle = rdtsc(); + uint64_t delta_cycles = deadline - MIN(deadline, curr_cycle); + uint64_t icr = (delta_cycles * APIC_TIMER_TSC_M) / APIC_TIMER_TSC_N; + + /* cap icr to 32 bits, and not zero */ + icr = CLAMP(icr, 1, UINT32_MAX); + x86_write_loapic(LOAPIC_TIMER_ICR, icr); + } +} + static void isr(const void *arg) { ARG_UNUSED(arg); @@ -94,7 +141,7 @@ static void isr(const void *arg) if (!IS_ENABLED(CONFIG_TICKLESS_KERNEL)) { uint64_t next_cycle = last_cycle + CYC_PER_TICK; - wrmsr(IA32_TSC_DEADLINE_MSR, next_cycle); + set_trigger(next_cycle); } k_spin_unlock(&lock, key); @@ -132,7 +179,7 @@ void sys_clock_set_timeout(int32_t ticks, bool idle) if (next_cycle < last_cycle) { next_cycle = UINT64_MAX; } - wrmsr(IA32_TSC_DEADLINE_MSR, next_cycle); + set_trigger(next_cycle); k_spin_unlock(&lock, key); } @@ -213,28 +260,44 @@ static int sys_clock_driver_init(void) #ifdef CONFIG_ASSERT uint32_t eax, ebx, ecx, edx; - ecx = 0; /* prevent compiler warning */ - __get_cpuid(CPUID_BASIC_INFO_1, &eax, &ebx, &ecx, &edx); - __ASSERT((ecx & BIT(24)) != 0, "No TSC Deadline support"); + if (IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER)) { + ecx = 0; /* prevent compiler warning */ + __get_cpuid(CPUID_BASIC_INFO_1, &eax, &ebx, &ecx, &edx); + __ASSERT((ecx & BIT(24)) != 0, "No TSC Deadline support"); + } edx = 0; /* prevent compiler warning */ __get_cpuid(0x80000007, &eax, &ebx, &ecx, &edx); __ASSERT((edx & BIT(8)) != 0, "No Invariant TSC support"); - ebx = 0; /* prevent compiler warning */ - __get_cpuid_count(CPUID_EXTENDED_FEATURES_LVL, 0, &eax, &ebx, &ecx, &edx); - __ASSERT((ebx & BIT(1)) != 0, "No TSC_ADJUST MSR support"); + if (IS_ENABLED(CONFIG_SMP)) { + ebx = 0; /* prevent compiler warning */ + __get_cpuid_count(CPUID_EXTENDED_FEATURES_LVL, 0, &eax, &ebx, &ecx, &edx); + __ASSERT((ebx & BIT(1)) != 0, "No TSC_ADJUST MSR support"); + } #endif - clear_tsc_adjust(); + if (IS_ENABLED(CONFIG_SMP)) { + clear_tsc_adjust(); + } /* Timer interrupt number is runtime-fetched, so can't use * static IRQ_CONNECT() */ irq_connect_dynamic(timer_irq(), CONFIG_APIC_TIMER_IRQ_PRIORITY, isr, 0, 0); + if (IS_ENABLED(CONFIG_APIC_TIMER_TSC)) { + uint32_t timer_conf; + + timer_conf = x86_read_loapic(LOAPIC_TIMER_CONFIG); + timer_conf &= ~0x0f; /* clear divider bits */ + timer_conf |= 0x0b; /* divide by 1 */ + x86_write_loapic(LOAPIC_TIMER_CONFIG, timer_conf); + } + lvt_reg.val = x86_read_loapic(LOAPIC_TIMER); - lvt_reg.lvt.mode = TSC_DEADLINE; + lvt_reg.lvt.mode = IS_ENABLED(CONFIG_APIC_TSC_DEADLINE_TIMER) ? + TSC_DEADLINE : ONE_SHOT; lvt_reg.lvt.masked = 0; x86_write_loapic(LOAPIC_TIMER, lvt_reg.val); @@ -248,7 +311,7 @@ static int sys_clock_driver_init(void) last_tick = rdtsc() / CYC_PER_TICK; last_cycle = last_tick * CYC_PER_TICK; if (!IS_ENABLED(CONFIG_TICKLESS_KERNEL)) { - wrmsr(IA32_TSC_DEADLINE_MSR, last_cycle + CYC_PER_TICK); + set_trigger(last_cycle + CYC_PER_TICK); } irq_enable(timer_irq()); diff --git a/soc/intel/apollo_lake/Kconfig.defconfig b/soc/intel/apollo_lake/Kconfig.defconfig index 7ea881ce59e..ef062600dfa 100644 --- a/soc/intel/apollo_lake/Kconfig.defconfig +++ b/soc/intel/apollo_lake/Kconfig.defconfig @@ -14,9 +14,6 @@ if APIC_TIMER config APIC_TIMER_IRQ default 24 -config APIC_TIMER_TSC - default y - endif # APIC_TIMER config X86_DYNAMIC_IRQ_STUBS