kernel: Add a "nocache" read-write memory section

Add a "nocache" read-write memory section that is configured to
not be cached. This memory section can be used to perform DMA
transfers when cache coherence issues are not optimal or can not
be solved using cache maintenance operations.

This is currently only supported on ARM Cortex M7 with MPU.

Fixes #2927

Signed-off-by: Aurelien Jarno <aurelien@aurel32.net>

arch/Kconfig

@@ -202,6 +202,18 @@ config SIMPLE_FATAL_ERROR_HANDLER
 	  for footprint-concerned systems. Only enable this option if you do not
 	  want debug capabilities in case of system fatal error.
 
+if ARCH_HAS_NOCACHE_MEMORY_SUPPORT
+
+config NOCACHE_MEMORY
+	bool "Support for uncached memory"
+	help
+	  Add a "nocache" read-write memory section that is configured to
+	  not be cached. This memory section can be used to perform DMA
+	  transfers when cache coherence issues are not optimal or can not
+	  be solved using cache maintenance operations.
+
+endif # ARCH_HAS_NOCACHE_MEMORY_SUPPORT
+
 menu "Interrupt Configuration"
 #
 # Interrupt related configs
@@ -281,6 +293,9 @@ config ARCH_HAS_USERSPACE
 config ARCH_HAS_EXECUTABLE_PAGE_BIT
 	bool
 
+config ARCH_HAS_NOCACHE_MEMORY_SUPPORT
+	bool
+
 #
 # Other architecture related options
 #

arch/arm/core/Kconfig

@@ -22,6 +22,7 @@ config CPU_CORTEX_M
 	select ARCH_HAS_TRUSTED_EXECUTION if ARM_TRUSTZONE_M
 	select ARCH_HAS_STACK_PROTECTION if ARM_MPU || CPU_CORTEX_M_HAS_SPLIM
 	select ARCH_HAS_USERSPACE if ARM_MPU
+	select ARCH_HAS_NOCACHE_MEMORY_SUPPORT if ARM_MPU && CPU_HAS_ARM_MPU && CPU_CORTEX_M7
 	help
 	  This option signifies the use of a CPU of the Cortex-M family.
 

arch/arm/core/cortex_m/mpu/arm_mpu.c

@@ -80,7 +80,7 @@ void arm_core_mpu_disable(void)
 }
 
 #if defined(CONFIG_USERSPACE) || defined(CONFIG_MPU_STACK_GUARD) || \
-	defined(CONFIG_APPLICATION_MEMORY)
+	defined(CONFIG_APPLICATION_MEMORY) || defined(CONFIG_NOCACHE_MEMORY)
 
 /**
  * This internal function is utilized by the MPU driver to parse the intent
@@ -93,6 +93,11 @@ static inline int _get_region_attr_by_type(arm_mpu_region_attr_t *p_attr,
 	u32_t type, u32_t base, u32_t size)
 {
 	switch (type) {
+#ifdef CONFIG_NOCACHE_MEMORY
+	case NOCACHE_MEMORY_REGION:
+		_get_mpu_ram_nocache_region_attr(p_attr, P_RW_U_NA, base, size);
+		return 0;
+#endif
 #ifdef CONFIG_USERSPACE
 	case THREAD_STACK_REGION:
 		_get_mpu_ram_region_attr(p_attr, P_RW_U_RW, base, size);
@@ -313,7 +318,7 @@ int arm_core_mpu_buffer_validate(void *addr, size_t size, int write)
 	return _mpu_buffer_validate(addr, size, write);
 }
 #endif /* CONFIG_USERSPACE */
-#endif /* USERSPACE || MPU_STACK_GUARD || APPLICATION_MEMORY */
+#endif /* USERSPACE || MPU_STACK_GUARD || APPLICATION_MEMORY || NOCACHE_MEMORY */
 
 /* ARM MPU Driver Initial Setup */
 
@@ -354,10 +359,24 @@ static int arm_mpu_init(struct device *arg)
 		_region_init(r_index, &mpu_config.mpu_regions[r_index]);
 	}
 
-#if defined(CONFIG_APPLICATION_MEMORY)
+#if defined(CONFIG_NOCACHE_MEMORY) || defined(CONFIG_APPLICATION_MEMORY)
 	u32_t index, size;
 	struct arm_mpu_region region_conf;
+#endif
 
+#if defined(CONFIG_NOCACHE_MEMORY)
+	/* configure non-cached memory */
+	index = _get_region_index_by_type(NOCACHE_MEMORY_REGION);
+	size = (u32_t)&_nocache_ram_end - (u32_t)&_nocache_ram_start;
+	_get_region_attr_by_type(&region_conf.attr, NOCACHE_MEMORY_REGION,
+			(u32_t)&_nocache_ram_start, size);
+	region_conf.base = (u32_t)&_nocache_ram_start;
+	if (size > 0) {
+		_region_init(index, &region_conf);
+	}
+#endif /* CONFIG_NOCACHE_MEMORY */
+
+#if defined(CONFIG_APPLICATION_MEMORY)
 	/* configure app data portion */
 	index = _get_region_index_by_type(THREAD_APP_DATA_REGION);
 	size = (u32_t)&__app_ram_end - (u32_t)&__app_ram_start;

arch/arm/core/cortex_m/mpu/arm_mpu_v7_internal.h

@@ -36,7 +36,7 @@ static void _region_init(u32_t index, const struct arm_mpu_region *region_conf)
 }
 
 #if defined(CONFIG_USERSPACE) || defined(CONFIG_MPU_STACK_GUARD) || \
-	defined(CONFIG_APPLICATION_MEMORY)
+	defined(CONFIG_APPLICATION_MEMORY) || defined(CONFIG_NOCACHE_MEMORY)
 
 static inline u8_t _get_num_regions(void);
 static inline u32_t _get_region_index_by_type(u32_t type);
@@ -90,6 +90,8 @@ static inline u32_t _get_region_attr(u32_t xn, u32_t ap, u32_t tex,
 		| (size));
 }
 
+#if defined(CONFIG_USERSPACE) || defined(CONFIG_MPU_STACK_GUARD) || \
+	defined(CONFIG_APPLICATION_MEMORY)
 /**
  * This internal function allocates default RAM cache-ability, share-ability,
  * and execution allowance attributes along with the requested access
@@ -105,6 +107,24 @@ static inline void _get_mpu_ram_region_attr(arm_mpu_region_attr_t *p_attr,
 
 	p_attr->rasr = _get_region_attr(1, ap, 1, 1, 1, 0, 0, size);
 }
+#endif /* USERSPACE || MPU_STACK_GUARD || APPLICATION_MEMORY */
+
+#if defined(CONFIG_NOCACHE_MEMORY)
+/**
+ * This internal function allocates non-cached, shareable, non-executable
+ * memory along with the requested access permissions ans size.
+ */
+static inline void _get_mpu_ram_nocache_region_attr(arm_mpu_region_attr_t *p_attr,
+	u32_t ap, u32_t base, u32_t size)
+{
+	/* in ARMv7-M MPU the base address is not required
+	 * to determine region attributes.
+	 */
+	(void) base;
+
+	p_attr->rasr = _get_region_attr(1, ap, 1, 0, 0, 1, 0, size);
+}
+#endif /* CONFIG_NO_CACHE_MEMORY */
 
 /**
  * This internal function is utilized by the MPU driver to combine a given
@@ -233,6 +253,7 @@ static inline int _mpu_buffer_validate(void *addr, size_t size, int write)
 }
 #endif /* CONFIG_USERSPACE */
 
-#endif /* USERSPACE || MPU_STACK_GUARD || APPLICATION_MEMORY */
+#endif /* USERSPACE || MPU_STACK_GUARD || APPLICATION_MEMORY
+		|| NOCACHE_MEMORY */
 
 #endif	/* ZEPHYR_ARCH_ARM_CORE_CORTEX_M_MPU_ARM_MPU_V7_INTERNAL_H_ */

include/arch/arm/cortex_m/mpu/arm_core_mpu_dev.h

@@ -32,6 +32,9 @@ extern "C" {
  *
  */
 enum {
+#ifdef CONFIG_NOCACHE_MEMORY
+	NOCACHE_MEMORY_REGION,
+#endif
 #ifdef CONFIG_APPLICATION_MEMORY
 	THREAD_APP_DATA_REGION,
 #endif

include/arch/arm/cortex_m/scripts/linker.ld

@@ -348,6 +348,20 @@ SECTIONS
 	__app_ram_size = __app_ram_end - __app_ram_start;
 #endif /* CONFIG_APPLICATION_MEMORY */
 
+#if defined(CONFIG_NOCACHE_MEMORY)
+	/* Non-cached region of RAM */
+	SECTION_PROLOGUE(_NOCACHE_SECTION_NAME,(NOLOAD),)
+	{
+		MPU_ALIGN(_nocache_ram_size);
+		_nocache_ram_start = .;
+		KERNEL_INPUT_SECTION(.nocache)
+		KERNEL_INPUT_SECTION(".nocache.*")
+		MPU_ALIGN(_nocache_ram_size);
+		_nocache_ram_end = .;
+	} GROUP_LINK_IN(RAMABLE_REGION)
+	_nocache_ram_size = _nocache_ram_end - _nocache_ram_start;
+#endif /* CONFIG_NOCACHE_MEMORY */
+
 #if defined(CONFIG_APP_SHARED_MEM)
 #define APP_SHARED_ALIGN . = ALIGN(_region_min_align);
 #if defined(CONFIG_MPU_REQUIRES_POWER_OF_TWO_ALIGNMENT)
@@ -371,7 +385,6 @@ SECTIONS
 	_app_smem_rom_start = LOADADDR(_APP_SMEM_SECTION_NAME);
 #endif  /* CONFIG_APP_SHARED_MEM */
 
-
     SECTION_DATA_PROLOGUE(_BSS_SECTION_NAME,(NOLOAD),)
 	{
         /*

include/linker/linker-defs.h

@@ -270,6 +270,19 @@ extern char __sg_end[];
 extern char __sg_size[];
 #endif /* CONFIG_ARM_FIRMWARE_HAS_SECURE_ENTRY_FUNCS */
 
+/*
+ * Non-cached kernel memory region, currently only available on ARM Cortex-M7
+ * with a MPU. Start and end will be aligned for memory management/protection
+ * hardware for the target architecture.
+ *
+ * All the functions with '__nocache' keyword will be placed into this
+ * section.
+ */
+#ifdef CONFIG_NOCACHE_MEMORY
+extern char _nocache_ram_start[];
+extern char _nocache_ram_end[];
+extern char _nocache_ram_size[];
+#endif /* CONFIG_NOCACHE_MEMORY */
 
 #endif /* ! _ASMLANGUAGE */
 

include/linker/section_tags.h

@@ -25,6 +25,10 @@
 #define __ccm_noinit_section _GENERIC_SECTION(_CCM_NOINIT_SECTION_NAME)
 #endif /* CONFIG_ARM */
 
+#if defined(CONFIG_NOCACHE_MEMORY)
+#define __nocache __in_section_unique(_NOCACHE_SECTION_NAME)
+#endif /* CONFIG_NOCACHE_MEMORY */
+
 #endif /* !_ASMLANGUAGE */
 
 #endif /* ZEPHYR_INCLUDE_LINKER_SECTION_TAGS_H_ */

include/linker/sections.h

@@ -56,7 +56,10 @@
 #define _CCM_DATA_SECTION_NAME		.ccm_data
 #define _CCM_BSS_SECTION_NAME		.ccm_bss
 #define _CCM_NOINIT_SECTION_NAME	.ccm_noinit
+#endif
 
+#ifdef CONFIG_NOCACHE_MEMORY
+#define _NOCACHE_SECTION_NAME nocache
 #endif
 
 #include <linker/section_tags.h>