From 334c14e66ecac9710172a0d4f4a63a3ba19e98fd Mon Sep 17 00:00:00 2001 From: Benjamin Walsh Date: Fri, 18 Sep 2015 16:36:57 -0400 Subject: [PATCH] x86: CLFLUSH and cache line size detection Detect the presence of CLFLUSH instruction and cache line size at runtime. It is still possible to set them manually via kconfig options if the values are known. Change-Id: I00bda1de4c5c241826ead6f43b887b99a963cc7b Signed-off-by: Benjamin Walsh --- Makefile.inc | 1 - arch/x86/Kconfig | 81 ++++++++++++++++++++++++++------ arch/x86/core/cache.c | 70 +++++++++++++++++++++++---- arch/x86/core/cache_s.S | 51 ++++++++++++++++++-- arch/x86/include/cache_private.h | 26 ++++++++++ include/cache.h | 39 +++++++++++++++ 6 files changed, 239 insertions(+), 29 deletions(-) create mode 100644 arch/x86/include/cache_private.h create mode 100644 include/cache.h diff --git a/Makefile.inc b/Makefile.inc index 2adb4e11544..b8cf1672460 100644 --- a/Makefile.inc +++ b/Makefile.inc @@ -83,5 +83,4 @@ pristine: PHONY += FORCE initconfig FORCE: - .PHONY: $(PHONY) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 3b167f17140..67816ff1e30 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -65,32 +65,25 @@ default CPU_MINUTEIA config CPU_ATOM bool "Atom" select CMOV + select CPU_MIGHT_SUPPORT_CLFLUSH if CACHE_FLUSHING help This option signifies the use of a CPU from the Atom family. config CPU_MINUTEIA bool "Minute IA" + select CPU_MIGHT_SUPPORT_CLFLUSH if CACHE_FLUSHING help This option signifies the use of a CPU from the Minute IA family. endchoice -config CPU_MIGHT_SUPPORT_CLFLUSH +config CACHE_FLUSHING bool default n + prompt "Enable cache flushing mechanism" help - If a platform uses a processor that possibly implements CLFLUSH, change - the default in that platform's config file. - -#FIXME This option is not being used in the current code base. -config CLFLUSH_INSTRUCTION_SUPPORTED - bool - prompt "CLFLUSH instruction supported" if CPU_MIGHT_SUPPORT_CLFLUSH - depends on CPU_MIGHT_SUPPORT_CLFLUSH - default n - help - Only enable this if the CLFLUSH instruction is supported, so that - an implementation of sys_cache_flush() that uses CLFLUSH is made - available, instead of the one using WBINVD. + This links in the sys_cache_flush() function. A mechanism for flushing the + cache must be selected as well. By default, that mechanism is discovered at + runtime. menu "Platform Capabilities" config ADVANCED_IDLE_SUPPORTED @@ -167,13 +160,71 @@ config CMOV This option signifies the use of an Intel CPU that supports the CMOV instruction. +config CACHE_LINE_SIZE_DETECT + bool + prompt "Detect cache line size at runtime" + default y + help + This option enables querying the CPUID register for finding the cache line + size at the expense of taking more memory and code and a slightly increased + boot time. + + If the CPU's cache line size is known in advance, disable this option and + manually enter the value for CACHE_LINE_SIZE. + config CACHE_LINE_SIZE - int "Cache line size" + int + prompt "Cache line size" if !CACHE_LINE_SIZE_DETECT + default 0 if CACHE_LINE_SIZE_DETECT default 64 if CPU_ATOM default 0 help Size in bytes of a CPU cache line. + Detect automatically at runtime by selecting CACHE_LINE_SIZE_DETECT. + +config CPU_MIGHT_SUPPORT_CLFLUSH + bool + depends on CACHE_FLUSHING + default n + help + If a platform uses a processor that possibly implements CLFLUSH, change + the default in that platform's config file. + +config CLFLUSH_INSTRUCTION_SUPPORTED + bool + prompt "CLFLUSH instruction supported" if CPU_MIGHT_SUPPORT_CLFLUSH + depends on CPU_MIGHT_SUPPORT_CLFLUSH && !CLFLUSH_DETECT + default n + help + An implementation of sys_cache_flush() that uses CLFLUSH is made + available, instead of the one using WBINVD. + + This option should only be enabled if it is known in advance that the + CPU supports the CLFLUSH instruction. It disables runtime detection of + CLFLUSH support thereby reducing both memory footprint and boot time. + +config CLFLUSH_DETECT + bool + prompt "Detect support of CLFLUSH instruction at runtime" + depends on CPU_MIGHT_SUPPORT_CLFLUSH + default y + help + This option should be enabled if it is not known in advance whether the + CPU supports the CLFLUSH instruction or not. + + The CPU is queried at boot time to determine which of the multiple + implementations of sys_cache_flush() linked into the image is the + correct one to use. + + If the CPU's support (or lack thereof) of CLFLUSH is known in advance, then + disable this option and set CLFLUSH_INSTRUCTION_SUPPORTED as appropriate. + +config ARCH_CACHE_FLUSH_DETECT + bool + default y + depends on CLFLUSH_DETECT + endmenu menu "Floating Point Options" diff --git a/arch/x86/core/cache.c b/arch/x86/core/cache.c index c0afaedfbd9..94d67410d9d 100644 --- a/arch/x86/core/cache.c +++ b/arch/x86/core/cache.c @@ -23,39 +23,91 @@ #include #include #include +#include +#include +#include -#ifdef CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED +#if defined(CONFIG_CACHE_FLUSHING) -#if (CONFIG_CACHE_LINE_SIZE == 0) +#if defined(CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED) || \ + defined(CONFIG_CLFLUSH_DETECT) + +#if (CONFIG_CACHE_LINE_SIZE == 0) && !defined(CONFIG_CACHE_LINE_SIZE_DETECT) #error Cannot use this implementation with a cache line size of 0 #endif /** * - * @brief Flush a page to main memory + * @brief Flush cache lines to main memory * * No alignment is required for either or , but since * sys_cache_flush() iterates on the cache lines, a cache line alignment for * both is optimal. * - * The cache line size is specified via the CONFIG_CACHE_LINE_SIZE kconfig - * option. + * The cache line size is specified either via the CONFIG_CACHE_LINE_SIZE + * kconfig option or it is detected at runtime. * * @return N/A */ -void sys_cache_flush(vaddr_t virt, size_t size) +_sys_cache_flush_sig(_cache_flush_clflush) { int end; - size = ROUND_UP(size, CONFIG_CACHE_LINE_SIZE); + size = ROUND_UP(size, sys_cache_line_size); end = virt + size; - for (; virt < end; virt += CONFIG_CACHE_LINE_SIZE) { + for (; virt < end; virt += sys_cache_line_size) { __asm__ volatile("clflush %0;\n\t" : : "m"(virt)); } __asm__ volatile("mfence;\n\t"); } -#endif /* CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED */ +#endif /* CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED || CLFLUSH_DETECT */ + +#if defined(CONFIG_CLFLUSH_DETECT) || defined(CONFIG_CACHE_LINE_SIZE_DETECT) + +#include + +#if defined(CONFIG_CLFLUSH_DETECT) +_sys_cache_flush_t *sys_cache_flush; +static void init_cache_flush(void) +{ + if (_is_clflush_available()) { + sys_cache_flush = _cache_flush_clflush; + } else { + sys_cache_flush = _cache_flush_wbinvd; + } +} +#else +#define init_cache_flush() do { } while ((0)) +FUNC_ALIAS(_cache_flush_clflush, sys_cache_flush, void); +#endif + +#endif /* CACHE_FLUSHING */ + +#if defined(CONFIG_CACHE_LINE_SIZE_DETECT) +size_t sys_cache_line_size; +static void init_cache_line_size(void) +{ + sys_cache_line_size = _cache_line_size_get(); +} +#else +#define init_cache_line_size() do { } while ((0)) +#endif + +static int init_cache(struct device *unused) +{ + ARG_UNUSED(unused); + + init_cache_flush(); + init_cache_line_size(); + + return 0; +} + +DECLARE_DEVICE_INIT_CONFIG(cache, "", init_cache, NULL); +pre_kernel_early_init(cache, NULL); + +#endif /* CONFIG_CLFLUSH_DETECT || CONFIG_CACHE_LINE_SIZE_DETECT */ diff --git a/arch/x86/core/cache_s.S b/arch/x86/core/cache_s.S index 65c20742492..30381397cc0 100644 --- a/arch/x86/core/cache_s.S +++ b/arch/x86/core/cache_s.S @@ -20,13 +20,33 @@ DESCRIPTION This module contains functions for manipulating caches. */ -#ifndef CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED - #define _ASMLANGUAGE #include +#ifndef CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED + +#if defined(CONFIG_CLFLUSH_DETECT) + + #define CACHE_FLUSH_NAME _cache_flush_wbinvd + #define CPUID_CFLSH_BIT (1 << 19) + + GTEXT(_is_clflush_available) + +SECTION_FUNC(TEXT, _is_clflush_available) + pushl %ebx + movl $1, %eax + cpuid + movl %edx, %eax + andl $CPUID_CFLSH_BIT, %eax + popl %ebx + ret + +#else + #define CACHE_FLUSH_NAME sys_cache_flush +#endif + /* externs (internal APIs) */ - GTEXT(sys_cache_flush) + GTEXT(CACHE_FLUSH_NAME) /** * @@ -43,8 +63,31 @@ This module contains functions for manipulating caches. * @return N/A */ -SECTION_FUNC(TEXT, sys_cache_flush) +SECTION_FUNC(TEXT, CACHE_FLUSH_NAME) wbinvd ret #endif /* !CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED */ + +#if defined(CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED) || \ + defined(CONFIG_CLFLUSH_DETECT) + +#if defined(CONFIG_CACHE_LINE_SIZE_DETECT) + + #define CPUID_CACHE_LINE_MASK (0xff << 8) + + GTEXT(_cache_line_size_get) + +SECTION_FUNC(TEXT, _cache_line_size_get) + pushl %ebx + movl $1, %eax + cpuid + movl %ebx, %eax + andl $CPUID_CACHE_LINE_MASK, %eax + shrl $5,%eax /* shift right 8 to get value, then multiple by 8 + * to get cache line size */ + popl %ebx + ret + +#endif /* CONFIG_CACHE_LINE_SIZE_DETECT */ +#endif /* CONFIG_CLFLUSH_INSTRUCTION_SUPPORTED || CONFIG_CLFLUSH_DETECT */ diff --git a/arch/x86/include/cache_private.h b/arch/x86/include/cache_private.h new file mode 100644 index 00000000000..503c5499b7f --- /dev/null +++ b/arch/x86/include/cache_private.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2015 Wind River Systems, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _cache_private__h_ +#define _cache_private__h_ + +#include + +extern int _is_clflush_available(void); +extern void _cache_flush_wbinvd(vaddr_t, size_t); +extern size_t _cache_line_size_get(void); + +#endif /* _cache_private__h_ */ diff --git a/include/cache.h b/include/cache.h new file mode 100644 index 00000000000..dafec19620c --- /dev/null +++ b/include/cache.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2015 Wind River Systems, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _cache__h_ +#define _cache__h_ + +#include + +#if defined(CONFIG_CACHE_FLUSHING) +#define _sys_cache_flush_sig(x) void (x)(vaddr_t virt, size_t size) + +#if defined(CONFIG_ARCH_CACHE_FLUSH_DETECT) + typedef _sys_cache_flush_sig(_sys_cache_flush_t); + extern _sys_cache_flush_t *sys_cache_flush; +#else + extern _sys_cache_flush_sig(sys_cache_flush); +#endif +#endif /* CACHE_FLUSHING */ + +#if defined(CONFIG_CACHE_LINE_SIZE_DETECT) + extern size_t sys_cache_line_size; +#else + #define sys_cache_line_size CONFIG_CACHE_LINE_SIZE +#endif + +#endif /* _cache__h_ */