clock: add k_cycle_get_64

This change adds `k_cycle_get_64()` on platforms that
support a 64-bit cycle counter.

The interface functions `arch_k_cycle_get_64()` and
`sys_clock_cycle_get_64()` are also introduced.

Fixes #39934

Signed-off-by: Christopher Friedt <chrisfriedt@gmail.com>

doc/reference/kernel/timing/clocks.rst

@@ -19,8 +19,8 @@ the advantages of being universally portable and pervasively
 understood, though they may not match the precision of the underlying
 hardware perfectly.
 
-The kernel presents a "cycle" count via the
+The kernel presents a "cycle" count via the :c:func:`k_cycle_get_32`
-:c:func:`k_cycle_get_32` API.  The intent is that this counter
+and :c:func:`k_cycle_get_64` APIs.  The intent is that this counter
 represents the fastest cycle counter that the operating system is able
 to present to the user (for example, a CPU cycle counter) and that the
 read operation is very fast.  The expectation is that very sensitive

drivers/timer/Kconfig

@@ -7,6 +7,12 @@
 
 menu "Timer Drivers"
 
+config TIMER_HAS_64BIT_CYCLE_COUNTER
+	bool
+	help
+	  When this option is true, the k_cycle_get_64() call is
+	  available to provide values from a 64-bit cycle counter.
+
 menuconfig APIC_TIMER
 	bool "New local APIC timer"
 	depends on X86
@@ -34,6 +40,7 @@ config APIC_TIMER_IRQ
 
 config APIC_TIMER_TSC
 	bool "Use invariant TSC for sys_clock_cycle_get_32()"
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  If your CPU supports invariant TSC, and you know the ratio of the
 	  TSC frequency to CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC (the local APIC
@@ -84,6 +91,7 @@ config HPET_TIMER
 	select LOAPIC if X86
 	imply TIMER_READS_ITS_FREQUENCY_AT_RUNTIME
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This option selects High Precision Event Timer (HPET) as a
 	  system timer.
@@ -110,6 +118,7 @@ config ARM_ARCH_TIMER
 	depends on GIC
 	select ARCH_HAS_CUSTOM_BUSY_WAIT
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for the ARM architected
 	  timer which provides per-cpu timers attached to a GIC to deliver its
@@ -220,6 +229,7 @@ config RISCV_MACHINE_TIMER
 	bool "RISCV Machine Timer"
 	depends on SOC_FAMILY_RISCV_PRIVILEGE
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for the generic RISCV machine
 	  timer driver. It provides the standard "system clock driver" interfaces.
@@ -238,6 +248,7 @@ config LITEX_TIMER
 	bool "LiteX Timer"
 	default y
 	depends on SOC_RISCV32_LITEX_VEXRISCV
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for LiteX Timer.
 
@@ -246,6 +257,7 @@ config NATIVE_POSIX_TIMER
 	default y
 	depends on BOARD_NATIVE_POSIX
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for the native_posix HW timer
 	  model
@@ -264,6 +276,7 @@ config ESP32C3_SYS_TIMER
 	depends on SOC_ESP32C3
 	default y
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This option enables the system timer driver for the Espressif ESP32C3
 	  and provides the standard "system clock driver" interface.
@@ -303,6 +316,7 @@ config CC13X2_CC26X2_RTC_TIMER
 	bool "TI SimpleLink CC13x2/CC26x2 RTC timer"
 	depends on SOC_SERIES_CC13X2_CC26X2
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for the TI SimpleLink
 	  CC13X2_CC26X2 series Real Time Counter and provides the standard
@@ -339,6 +353,7 @@ config CAVS_TIMER
 	bool "CAVS DSP Wall Clock Timer on Intel SoC"
 	depends on CAVS_ICTL
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  The DSP wall clock timer is a timer driven directly by
 	  external oscillator and is external to the CPU core(s).
@@ -359,6 +374,7 @@ config NPCX_ITIM_TIMER
 	default y
 	depends on SOC_FAMILY_NPCX
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for the Nuvoton NPCX
 	  series internal 64/32-bit timers and provides the standard "system
@@ -413,6 +429,7 @@ config MCUX_OS_TIMER
 	depends on HAS_MCUX_OS_TIMER
 	default $(dt_compat_enabled,$(DT_COMPAT_NXP_OS_TIMER))
 	select TICKLESS_CAPABLE
+	select TIMER_HAS_64BIT_CYCLE_COUNTER
 	help
 	  This module implements a kernel device driver for the NXP OS
 	  event timer and provides the standard "system clock driver" interfaces.

drivers/timer/apic_tsc.c

@@ -97,6 +97,11 @@ uint32_t sys_clock_cycle_get_32(void)
 	return (uint32_t) rdtsc();
 }
 
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return rdtsc();
+}
+
 static inline uint32_t timer_irq(void)
 {
 	/* The Zephyr APIC API is... idiosyncratic.  The timer is a

drivers/timer/arm_arch_timer.c

@@ -153,6 +153,11 @@ uint32_t sys_clock_cycle_get_32(void)
 	return (uint32_t)arm_arch_timer_count();
 }
 
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return arm_arch_timer_count();
+}
+
 #ifdef CONFIG_ARCH_HAS_CUSTOM_BUSY_WAIT
 void arch_busy_wait(uint32_t usec_to_wait)
 {

drivers/timer/cavs_timer.c

@@ -77,6 +77,17 @@ static uint32_t count32(void)
 	return shim_regs->walclk32_lo;
 }
 
+static uint64_t count64(void)
+{
+	k_spinlock_key_t key = k_spin_lock(&lock);
+	uint64_t ret = shim_regs->walclk32_lo;
+
+	ret |= (uint64_t)shim_regs->walclk32_hi << 32;
+
+	k_spin_unlock(&lock, key);
+	return ret;
+}
+
 static void compare_isr(const void *arg)
 {
 	ARG_UNUSED(arg);
@@ -182,6 +193,11 @@ uint32_t sys_clock_cycle_get_32(void)
 	return count32();
 }
 
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return count64();
+}
+
 /* Runs on secondary cores */
 void smp_timer_init(void)
 {

drivers/timer/cc13x2_cc26x2_rtc_timer.c

@@ -240,6 +240,10 @@ uint32_t sys_clock_elapsed(void)
 
 uint32_t sys_clock_cycle_get_32(void)
 {
-	return (AONRTCCurrent64BitValueGet() / RTC_COUNTS_PER_CYCLE)
+	return (uint32_t)(AONRTCCurrent64BitValueGet() / RTC_COUNTS_PER_CYCLE);
-		& 0xFFFFFFFF;
+}
+
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return AONRTCCurrent64BitValueGet() / RTC_COUNTS_PER_CYCLE;
 }

drivers/timer/esp32c3_sys_timer.c

@@ -134,3 +134,15 @@ uint32_t sys_clock_cycle_get_32(void)
 {
 	return systimer_ll_get_counter_value_low(SYSTIMER_COUNTER_1);
 }
+
+uint64_t sys_clock_cycle_get_64(void)
+{
+	k_spinlock_key_t key = k_spin_lock(&lock);
+	uint64_t ret = systimer_ll_get_counter_value_low(SYSTIMER_COUNTER_1);
+
+	ret |= (uint64_t)systimer_ll_get_counter_value_high(SYSTIMER_COUNTER_1) << 32;
+
+	k_spin_unlock(&lock, key);
+
+	return ret;
+}

drivers/timer/hpet.c

@@ -438,6 +438,12 @@ uint32_t sys_clock_cycle_get_32(void)
 	return (uint32_t)hpet_counter_get();
 }
 
+__pinned_func
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return hpet_counter_get();
+}
+
 __pinned_func
 void sys_clock_idle_exit(void)
 {

drivers/timer/litex_timer.c

@@ -52,6 +52,20 @@ uint32_t sys_clock_cycle_get_32(void)
 	return timer_total;
 }
 
+uint64_t sys_clock_cycle_get_64(void)
+{
+	static struct k_spinlock lock;
+	uint64_t timer_total;
+	k_spinlock_key_t key = k_spin_lock(&lock);
+
+	litex_write8(UPDATE_TOTAL, TIMER_TOTAL_UPDATE);
+	timer_total = litex_read64(TIMER_TOTAL);
+
+	k_spin_unlock(&lock, key);
+
+	return timer_total;
+}
+
 /* tickless kernel is not supported */
 uint32_t sys_clock_elapsed(void)
 {

drivers/timer/mcux_os_timer.c

@@ -127,3 +127,8 @@ uint32_t sys_clock_cycle_get_32(void)
 {
 	return (uint32_t)OSTIMER_GetCurrentTimerValue(base);
 }
+
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return OSTIMER_GetCurrentTimerValue(base);
+}

drivers/timer/native_posix_timer.c

@@ -32,6 +32,11 @@ uint32_t sys_clock_cycle_get_32(void)
 	return hwm_get_time();
 }
 
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return hwm_get_time();
+}
+
 /**
  * Interrupt handler for the timer interrupt
  * Announce to the kernel that a number of ticks have passed

drivers/timer/npcx_itim_timer.c

@@ -267,6 +267,17 @@ uint32_t sys_clock_cycle_get_32(void)
 	return (uint32_t)(current);
 }
 
+uint64_t sys_clock_cycle_get_64(void)
+{
+	k_spinlock_key_t key = k_spin_lock(&lock);
+	uint64_t current = npcx_itim_get_sys_cyc64();
+
+	k_spin_unlock(&lock, key);
+
+	/* Return how many cycles since system kernel timer start counting */
+	return current;
+}
+
 int sys_clock_driver_init(const struct device *dev)
 {
 	ARG_UNUSED(dev);

drivers/timer/riscv_machine_timer.c

@@ -147,3 +147,8 @@ uint32_t sys_clock_cycle_get_32(void)
 {
 	return (uint32_t)mtime();
 }
+
+uint64_t sys_clock_cycle_get_64(void)
+{
+	return mtime();
+}

include/arch/arc/v2/misc.h

@@ -27,6 +27,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 {
 	return sys_clock_cycle_get_32();
 }
+
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
 #endif
 
 #ifdef __cplusplus

include/arch/arm/aarch32/misc.h

@@ -26,6 +26,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 static ALWAYS_INLINE void arch_nop(void)
 {
 	__asm__ volatile("nop");

include/arch/arm64/misc.h

@@ -27,6 +27,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 static ALWAYS_INLINE void arch_nop(void)
 {
 	__asm__ volatile("nop");

include/arch/nios2/arch.h

@@ -178,6 +178,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 static ALWAYS_INLINE void arch_nop(void)
 {
 	__asm__ volatile("nop");

include/arch/posix/arch.h

@@ -51,6 +51,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 static ALWAYS_INLINE void arch_nop(void)
 {
 	__asm__ volatile("nop");

include/arch/riscv/arch.h

@@ -354,6 +354,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 #ifdef CONFIG_USERSPACE
 #include <arch/riscv/error.h>
 #endif /* CONFIG_USERSPACE */

include/arch/sparc/arch.h

@@ -100,6 +100,12 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
 
 struct __esf {
 	uint32_t out[8];

include/arch/x86/arch.h

@@ -255,6 +255,14 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+__pinned_func
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 static ALWAYS_INLINE bool arch_irq_unlocked(unsigned int key)
 {
 	return (key & 0x200) != 0;

include/arch/xtensa/arch.h

@@ -63,6 +63,13 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return sys_clock_cycle_get_32();
 }
 
+extern uint64_t sys_clock_cycle_get_64(void);
+
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return sys_clock_cycle_get_64();
+}
+
 static ALWAYS_INLINE void arch_nop(void)
 {
 	__asm__ volatile("nop");

include/kernel.h

@@ -1639,6 +1639,27 @@ static inline uint32_t k_cycle_get_32(void)
 	return arch_k_cycle_get_32();
 }
 
+/**
+ * @brief Read the 64-bit hardware clock.
+ *
+ * This routine returns the current time in 64-bits, as measured by the
+ * system's hardware clock, if available.
+ *
+ * @see CONFIG_TIMER_HAS_64BIT_CYCLE_COUNTER
+ *
+ * @return Current hardware clock up-counter (in cycles).
+ */
+static inline uint64_t k_cycle_get_64(void)
+{
+	if (!IS_ENABLED(CONFIG_TIMER_HAS_64BIT_CYCLE_COUNTER)) {
+		__ASSERT(0, "64-bit cycle counter not enabled on this platform. "
+			    "See CONFIG_TIMER_HAS_64BIT_CYCLE_COUNTER");
+		return 0;
+	}
+
+	return arch_k_cycle_get_64();
+}
+
 /**
  * @}
  */

include/sys/arch_interface.h

@@ -58,6 +58,13 @@ typedef void (*k_thread_entry_t)(void *p1, void *p2, void *p3);
  */
 static inline uint32_t arch_k_cycle_get_32(void);
 
+/**
+ * Obtain the current cycle count, in units that are hardware-specific
+ *
+ * @see k_cycle_get_64()
+ */
+static inline uint64_t arch_k_cycle_get_64(void);
+
 /** @} */
 
 

subsys/testsuite/ztest/include/arch/cpu.h

@@ -27,6 +27,11 @@ static inline uint32_t arch_k_cycle_get_32(void)
 	return 0;
 }
 
+static inline uint64_t arch_k_cycle_get_64(void)
+{
+	return 0;
+}
+
 static ALWAYS_INLINE unsigned int arch_irq_lock(void)
 {
 	return 0;