diff --git a/subsys/bluetooth/CMakeLists.txt b/subsys/bluetooth/CMakeLists.txt
index 0722d5bc442..8650d86fcee 100644
--- a/subsys/bluetooth/CMakeLists.txt
+++ b/subsys/bluetooth/CMakeLists.txt
@@ -4,7 +4,12 @@ target_include_directories(subsys__bluetooth INTERFACE ${CMAKE_CURRENT_SOURCE_DI
 
 add_subdirectory(common)
 add_subdirectory_ifdef(CONFIG_BT_HCI host)
-add_subdirectory_ifdef(CONFIG_BT_CTLR controller)
 add_subdirectory_ifdef(CONFIG_BT_SHELL shell)
 
+if(CONFIG_BT_CTLR)
+  if(CONFIG_BT_LL_SW)
+    add_subdirectory(controller)
+  endif()
+endif()
+
 target_link_libraries(subsys__bluetooth INTERFACE zephyr_interface)
diff --git a/subsys/bluetooth/controller/CMakeLists.txt b/subsys/bluetooth/controller/CMakeLists.txt
index b7ef64d8558..f324191933e 100644
--- a/subsys/bluetooth/controller/CMakeLists.txt
+++ b/subsys/bluetooth/controller/CMakeLists.txt
@@ -7,6 +7,8 @@ zephyr_library_sources(
   ticker/ticker.c
   ll_sw/ll_addr.c
   ll_sw/ll_tx_pwr.c
+  ll_sw/ctrl.c
+  ll_sw/ll.c
   hci/hci_driver.c
   hci/hci.c
   )
@@ -16,28 +18,25 @@ zephyr_library_sources_ifdef(
   crypto/crypto.c
   )
 
-if(CONFIG_BT_LL_SW)
-  zephyr_library_sources(
-    ll_sw/ctrl.c
-    ll_sw/ll.c
-    )
-  zephyr_library_sources_ifdef(
-    CONFIG_BT_BROADCASTER
-    ll_sw/ll_adv.c
-    )
-  zephyr_library_sources_ifdef(
-    CONFIG_BT_OBSERVER
-    ll_sw/ll_scan.c
-    )
-  zephyr_library_sources_ifdef(
-    CONFIG_BT_CENTRAL
-    ll_sw/ll_master.c
-    )
-  zephyr_library_sources_ifdef(
-    CONFIG_BT_CTLR_DTM
-    ll_sw/ll_test.c
-    )
-endif()
+zephyr_library_sources_ifdef(
+  CONFIG_BT_BROADCASTER
+  ll_sw/ll_adv.c
+  )
+
+zephyr_library_sources_ifdef(
+  CONFIG_BT_OBSERVER
+  ll_sw/ll_scan.c
+  )
+
+zephyr_library_sources_ifdef(
+  CONFIG_BT_CENTRAL
+  ll_sw/ll_master.c
+  )
+
+zephyr_library_sources_ifdef(
+  CONFIG_BT_CTLR_DTM
+  ll_sw/ll_test.c
+  )
 
 zephyr_library_sources_ifdef(
   CONFIG_BT_CTLR_FILTER
diff --git a/subsys/bluetooth/controller/Kconfig b/subsys/bluetooth/controller/Kconfig
index bdd0af15cc1..19b76094a23 100644
--- a/subsys/bluetooth/controller/Kconfig
+++ b/subsys/bluetooth/controller/Kconfig
@@ -9,7 +9,6 @@ comment "BLE Controller support"
 
 config BT_CTLR
 	bool "Bluetooth Controller"
-	select BT_RECV_IS_RX_THREAD
 	help
 	  Enables support for SoC native controller implementations.
 
@@ -23,6 +22,7 @@ choice
 
 config BT_LL_SW
 	bool "Software-based BLE Link Layer"
+	select BT_RECV_IS_RX_THREAD
 	select ENTROPY_GENERATOR
 	select ENTROPY_NRF5_RNG if SOC_FAMILY_NRF
 	select ENTROPY_NRF5_BIAS_CORRECTION if SOC_FAMILY_NRF
@@ -560,8 +560,6 @@ config BT_CTLR_PA_LNA_GPIOTE_CHAN
 	help
 	  Select the nRF5 GPIOTE channel to use for PA/LNA GPIO feature.
 
-endif # BT_LL_SW
-
 comment "BLE Controller debug configuration"
 
 config BT_CTLR_ASSERT_HANDLER
@@ -590,4 +588,6 @@ config BT_CTLR_DEBUG_PINS
 	  when debugging with a logic analyzer or profiling certain sections of
 	  the code.
 
+endif # BT_LL_SW
+
 endif # BT_CTLR