diff --git a/boards/arm/mimxrt1020_evk/Kconfig.defconfig b/boards/arm/mimxrt1020_evk/Kconfig.defconfig
index 4893097b5a3..a7172e27149 100644
--- a/boards/arm/mimxrt1020_evk/Kconfig.defconfig
+++ b/boards/arm/mimxrt1020_evk/Kconfig.defconfig
@@ -27,12 +27,4 @@ config UART_MCUX_LPUART_1
 
 endif # UART_MCUX_LPUART
 
-if CODE_QSPI
-
-# Reserve space for the IVT
-config TEXT_SECTION_OFFSET
-	default 0x2000
-
-endif
-
 endif # BOARD_MIMXRT1020_EVK
diff --git a/boards/arm/mimxrt1050_evk/Kconfig.defconfig b/boards/arm/mimxrt1050_evk/Kconfig.defconfig
index 1a3b1937011..e84215cc8c8 100644
--- a/boards/arm/mimxrt1050_evk/Kconfig.defconfig
+++ b/boards/arm/mimxrt1050_evk/Kconfig.defconfig
@@ -46,14 +46,6 @@ config UART_MCUX_LPUART_3
 
 endif # UART_MCUX_LPUART
 
-if CODE_HYPERFLASH || CODE_QSPI
-
-# Reserve space for the IVT
-config TEXT_SECTION_OFFSET
-	default 0x2000
-
-endif
-
 if NETWORKING
 
 config NET_L2_ETHERNET
diff --git a/boards/arm/mimxrt1060_evk/Kconfig.defconfig b/boards/arm/mimxrt1060_evk/Kconfig.defconfig
index 2d0ec1b9d4a..b8dfec07ab3 100644
--- a/boards/arm/mimxrt1060_evk/Kconfig.defconfig
+++ b/boards/arm/mimxrt1060_evk/Kconfig.defconfig
@@ -28,12 +28,4 @@ config UART_MCUX_LPUART_1
 
 endif # UART_MCUX_LPUART
 
-if CODE_HYPERFLASH || CODE_QSPI
-
-# Reserve space for the IVT
-config TEXT_SECTION_OFFSET
-	default 0x2000
-
-endif
-
 endif # BOARD_MIMXRT1060_EVK || BOARD_MIMXRT1060_EVK_HYPERFLASH
diff --git a/soc/arm/nxp_imx/rt/Kconfig.defconfig.series b/soc/arm/nxp_imx/rt/Kconfig.defconfig.series
index 05370e5645c..c1b3bd0a6ce 100644
--- a/soc/arm/nxp_imx/rt/Kconfig.defconfig.series
+++ b/soc/arm/nxp_imx/rt/Kconfig.defconfig.series
@@ -15,6 +15,9 @@ config NUM_IRQS
 	# must be >= the highest interrupt number used
 	default 160
 
+config TEXT_SECTION_OFFSET
+	default 0x2000 if BOOT_FLEXSPI_NOR || BOOT_SEMC_NOR
+
 source "soc/arm/nxp_imx/rt/Kconfig.defconfig.mimxrt*"
 
 endif # SOC_SERIES_IMX_RT