Bluetooth: Mesh: Introduce separate workq for ADV EXT

this PR is to make the host always send packets.

Signed-off-by: Lingao Meng <menglingao@xiaomi.com>

doc/releases/release-notes-4.0.rst

@@ -148,6 +148,11 @@ Bluetooth
 
 * HCI Drivers
 
+* Mesh
+
+  * Introduced a mesh-specific workqueue to increase reliability of the mesh messages
+    transmission. To get the old behavior enable :kconfig:option:`CONFIG_BT_MESH_WORKQ_SYS`.
+
 Boards & SoC Support
 ********************
 

subsys/bluetooth/mesh/Kconfig

@@ -103,6 +103,64 @@ menuconfig BT_MESH_ADV_EXT
 
 if BT_MESH_ADV_EXT
 
+choice BT_MESH_WORKQ_CONTEXT
+	prompt "Advertising thread selection"
+	default BT_MESH_WORKQ_MESH
+	help
+	  Defines a context for mesh messages transmission.
+
+config BT_MESH_WORKQ_MESH
+	bool "Mesh-specific workqueue"
+	help
+	  When this option is selected, the mesh sends messages from the
+	  mesh-specific workqueue. This will ensure that messages are always sent.
+	  The application needs to ensure the mesh-specific workqueue size is large
+	  enough. Refer to BT_MESH_ADV_STACK_SIZE for the recommended minimum.
+
+config BT_MESH_WORKQ_SYS
+	bool "System workqueue"
+	help
+	  When this option is selected, the mesh sends messages from
+	  the system work queue. The application needs to ensure the system
+	  workqueue stack size (SYSTEM_WORKQUEUE_STACK_SIZE) is large enough,
+	  refer to BT_MESH_ADV_STACK_SIZE for the recommended minimum.
+
+	  When this option is enabled and the mesh tries to send a message,
+	  and the host ran out the HCI command buffers controlled by
+	  CONFIG_BT_BUF_CMD_TX_COUNT, the host returns -ENOBUFS immediately
+	  and the mesh drops the message transmission. To mitigate this
+	  issue, make sure to have sufficient number of HCI command buffers.
+	  When this option is enabled, the latency of sending mesh messages
+	  will be affected by other users on the system work queue, resulting in
+	  reduced reliability for sending mesh messages.
+
+endchoice
+
+if BT_MESH_WORKQ_MESH
+
+config BT_MESH_ADV_STACK_SIZE
+	int "Mesh extended advertiser thread stack size"
+	default 1536 if BT_MESH_PROXY
+	default 1024 if BT_HOST_CRYPTO
+	default 776 if BT_MESH_PRIV_BEACONS
+	default 768
+	help
+	  Size of bt mesh adv thread stack.
+
+	  NOTE: This is an advanced setting and should not be changed unless
+	  absolutely necessary
+
+config BT_MESH_ADV_PRIO
+	int "Mesh advertiser thread priority"
+	default 7
+	help
+	  Priority of bt mesh adv thread.
+
+	  NOTE: This is an advanced setting and should not be changed unless
+	  absolutely necessary
+
+endif # BT_MESH_WORKQ_MESH
+
 config BT_MESH_RELAY_ADV_SETS
 	int "Maximum of simultaneous relay message support"
 	default 0

subsys/bluetooth/mesh/adv.h

@@ -123,4 +123,6 @@ int bt_mesh_scan_active_set(bool active);
 int bt_mesh_adv_bt_data_send(uint8_t num_events, uint16_t adv_interval,
 			     const struct bt_data *ad, size_t ad_len);
 
+int bt_mesh_wq_submit(struct k_work *work);
+
 #endif /* ZEPHYR_SUBSYS_BLUETOOTH_MESH_ADV_H_ */

subsys/bluetooth/mesh/adv_ext.c

@@ -33,6 +33,14 @@ LOG_MODULE_REGISTER(bt_mesh_adv_ext);
 #define CONFIG_BT_MESH_RELAY_ADV_SETS 0
 #endif
 
+#ifdef CONFIG_BT_MESH_ADV_STACK_SIZE
+#define MESH_WORKQ_PRIORITY   CONFIG_BT_MESH_ADV_PRIO
+#define MESH_WORKQ_STACK_SIZE CONFIG_BT_MESH_ADV_STACK_SIZE
+#else
+#define MESH_WORKQ_PRIORITY   0
+#define MESH_WORKQ_STACK_SIZE 0
+#endif
+
 enum {
 	/** Controller is currently advertising */
 	ADV_FLAG_ACTIVE,
@@ -69,6 +77,15 @@ struct bt_mesh_ext_adv {
 static void send_pending_adv(struct k_work *work);
 static bool schedule_send(struct bt_mesh_ext_adv *ext_adv);
 
+static struct k_work_q bt_mesh_workq;
+static K_KERNEL_STACK_DEFINE(thread_stack, MESH_WORKQ_STACK_SIZE);
+
+#if defined(CONFIG_BT_MESH_WORKQ_MESH)
+#define MESH_WORKQ &bt_mesh_workq
+#else /* CONFIG_BT_MESH_WORKQ_SYS */
+#define MESH_WORKQ &k_sys_work_q
+#endif /* CONFIG_BT_MESH_WORKQ_MESH */
+
 static struct bt_mesh_ext_adv advs[] = {
 	[0] = {
 		.tags = (
@@ -258,7 +275,7 @@ static bool schedule_send_with_mask(struct bt_mesh_ext_adv *ext_adv, int ignore_
 	}
 
 	atomic_clear_bit(ext_adv->flags, ADV_FLAG_SCHEDULE_PENDING);
-	k_work_submit(&ext_adv->work);
+	bt_mesh_wq_submit(&ext_adv->work);
 
 	return true;
 }
@@ -407,7 +424,7 @@ int bt_mesh_adv_terminate(struct bt_mesh_adv *adv)
 
 		atomic_set_bit(ext_adv->flags, ADV_FLAG_SENT);
 
-		k_work_submit(&ext_adv->work);
+		bt_mesh_wq_submit(&ext_adv->work);
 
 		return 0;
 	}
@@ -429,6 +446,13 @@ void bt_mesh_adv_init(void)
 	for (int i = 0; i < ARRAY_SIZE(advs); i++) {
 		(void)memcpy(&advs[i].adv_param, &adv_param, sizeof(adv_param));
 	}
+
+	if (IS_ENABLED(CONFIG_BT_MESH_WORKQ_MESH)) {
+		k_work_queue_init(&bt_mesh_workq);
+		k_work_queue_start(&bt_mesh_workq, thread_stack, MESH_WORKQ_STACK_SIZE,
+				   K_PRIO_COOP(MESH_WORKQ_PRIORITY), NULL);
+		k_thread_name_set(&bt_mesh_workq.thread, "BT MESH WQ");
+	}
 }
 
 static struct bt_mesh_ext_adv *adv_instance_find(struct bt_le_ext_adv *instance)
@@ -458,7 +482,7 @@ static void adv_sent(struct bt_le_ext_adv *instance,
 
 	atomic_set_bit(ext_adv->flags, ADV_FLAG_SENT);
 
-	k_work_submit(&ext_adv->work);
+	bt_mesh_wq_submit(&ext_adv->work);
 }
 
 #if defined(CONFIG_BT_MESH_GATT_SERVER)
@@ -503,13 +527,13 @@ int bt_mesh_adv_enable(void)
 
 int bt_mesh_adv_disable(void)
 {
-	int err;
 	struct k_work_sync sync;
+	int err;
 
 	for (int i = 0; i < ARRAY_SIZE(advs); i++) {
 		atomic_set_bit(advs[i].flags, ADV_FLAG_SUSPENDING);
 
-		if (k_current_get() != &k_sys_work_q.thread ||
+		if (k_current_get() != k_work_queue_thread_get(MESH_WORKQ) ||
 		    (k_work_busy_get(&advs[i].work) & K_WORK_RUNNING) == 0) {
 			k_work_flush(&advs[i].work, &sync);
 		}
@@ -562,3 +586,8 @@ int bt_mesh_adv_bt_data_send(uint8_t num_events, uint16_t adv_interval,
 {
 	return bt_data_send(advs, num_events, adv_interval, ad, ad_len);
 }
+
+int bt_mesh_wq_submit(struct k_work *work)
+{
+	return k_work_submit_to_queue(MESH_WORKQ, work);
+}

subsys/bluetooth/mesh/adv_legacy.c

@@ -277,3 +277,8 @@ int bt_mesh_adv_gatt_start(const struct bt_le_adv_param *param, int32_t duration
 	adv_timeout = duration;
 	return bt_le_adv_start(param, ad, ad_len, sd, sd_len);
 }
+
+int bt_mesh_wq_submit(struct k_work *work)
+{
+	return k_work_submit(work);
+}

subsys/bluetooth/mesh/proxy_msg.c

@@ -64,6 +64,15 @@ static void proxy_sar_timeout(struct k_work *work)
 	LOG_WRN("Proxy SAR timeout");
 
 	role = CONTAINER_OF(dwork, struct bt_mesh_proxy_role, sar_timer);
+
+	while (!k_fifo_is_empty(&role->pending)) {
+		struct bt_mesh_adv *adv = k_fifo_get(&role->pending, K_NO_WAIT);
+
+		__ASSERT_NO_MSG(adv);
+
+		bt_mesh_adv_unref(adv);
+	}
+
 	if (role->conn) {
 		bt_conn_disconnect(role->conn,
 				   BT_HCI_ERR_REMOTE_USER_TERM_CONN);
@@ -200,7 +209,7 @@ static void buf_send_end(struct bt_conn *conn, void *user_data)
 	bt_mesh_adv_unref(adv);
 }
 
-int bt_mesh_proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
+static int proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
 {
 	int err;
 
@@ -230,6 +239,41 @@ int bt_mesh_proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
 	return err;
 }
 
+int bt_mesh_proxy_relay_send(struct bt_conn *conn, struct bt_mesh_adv *adv)
+{
+	struct bt_mesh_proxy_role *role = &roles[bt_conn_index(conn)];
+
+	k_fifo_put(&role->pending, bt_mesh_adv_ref(adv));
+
+	bt_mesh_wq_submit(&role->work);
+
+	return 0;
+}
+
+static void proxy_msg_send_pending(struct k_work *work)
+{
+	struct bt_mesh_proxy_role *role;
+	struct k_work_delayable *dwork = k_work_delayable_from_work(work);
+	struct bt_mesh_adv *adv;
+
+	role = CONTAINER_OF(dwork, struct bt_mesh_proxy_role, sar_timer);
+	if (!role->conn) {
+		return;
+	}
+
+	adv = k_fifo_get(&role->pending, K_NO_WAIT);
+	if (!adv) {
+		return;
+	}
+
+	(void)proxy_relay_send(role->conn, adv);
+	bt_mesh_adv_unref(adv);
+
+	if (!k_fifo_is_empty(&role->pending)) {
+		bt_mesh_wq_submit(&role->work);
+	}
+}
+
 static void proxy_msg_init(struct bt_mesh_proxy_role *role)
 {
 	/* Check if buf has been allocated, in this way, we no longer need
@@ -247,6 +291,9 @@ static void proxy_msg_init(struct bt_mesh_proxy_role *role)
 
 	net_buf_simple_reset(&role->buf);
 
+	k_fifo_init(&role->pending);
+	k_work_init(&role->work, proxy_msg_send_pending);
+
 	k_work_init_delayable(&role->sar_timer, proxy_sar_timeout);
 }
 

subsys/bluetooth/mesh/proxy_msg.h

@@ -37,6 +37,9 @@ struct bt_mesh_proxy_role {
 	struct bt_conn *conn;
 	uint8_t msg_type;
 
+	struct k_fifo pending;
+	struct k_work work;
+
 	struct {
 		proxy_send_cb_t send;
 		proxy_recv_cb_t recv;

subsys/bluetooth/mesh/proxy_srv.c

@@ -916,7 +916,7 @@ static ssize_t proxy_ccc_write(struct bt_conn *conn,
 	client = find_client(conn);
 	if (client->filter_type == NONE) {
 		client->filter_type = ACCEPT;
-		k_work_submit(&client->send_beacons);
+		bt_mesh_wq_submit(&client->send_beacons);
 	}
 
 	return sizeof(value);

tests/bsim/bluetooth/mesh/compile.sh

@@ -15,12 +15,15 @@ app=tests/bsim/bluetooth/mesh conf_overlay=overlay_pst.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay=overlay_gatt.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay=overlay_low_lat.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay=overlay_psa.conf compile
+app=tests/bsim/bluetooth/mesh conf_overlay=overlay_workq_sys.conf compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_pst.conf;overlay_psa.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_gatt.conf;overlay_psa.conf" compile
+app=tests/bsim/bluetooth/mesh conf_overlay="overlay_gatt.conf;overlay_workq_sys.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_low_lat.conf;overlay_psa.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_gatt.conf;overlay_low_lat.conf" compile
 app=tests/bsim/bluetooth/mesh conf_overlay="overlay_pst.conf;overlay_gatt.conf" compile
 app=tests/bsim/bluetooth/mesh \
   conf_overlay="overlay_pst.conf;overlay_gatt.conf;overlay_psa.conf" compile
-
+app=tests/bsim/bluetooth/mesh \
+  conf_overlay="overlay_pst.conf;overlay_gatt.conf;overlay_workq_sys.conf" compile
 wait_for_background_jobs

tests/bsim/bluetooth/mesh/overlay_workq_sys.conf

@@ -0,0 +1 @@
+CONFIG_BT_MESH_WORKQ_SYS=y

tests/bsim/bluetooth/mesh/src/test_suspend.c

@@ -295,6 +295,9 @@ static void dut_pub_common(bool disable_bt)
 		ASSERT_OK_MSG(k_sem_take(&publish_sem, K_SECONDS(30)), "Pub timed out");
 	}
 
+	/* Allow publishing to finish before suspending. */
+	k_sleep(K_MSEC(100));
+
 	ASSERT_OK_MSG(bt_mesh_suspend(), "Failed to suspend Mesh.");
 
 	if (disable_bt) {

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/disable.sh

@@ -36,3 +36,6 @@ RunTest mesh_adv_disable adv_tx_disable adv_rx_disable
 # Low latency overlay uses legacy advertiser
 overlay=overlay_low_lat_conf
 RunTest mesh_adv_disable adv_tx_disable adv_rx_disable
+
+overlay=overlay_workq_sys_conf
+RunTest mesh_adv_disable_workq adv_tx_disable adv_rx_disable

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/proxy_mixin.sh

@@ -21,5 +21,8 @@ source $(dirname "${BASH_SOURCE[0]}")/../../_mesh_test.sh
 overlay=overlay_gatt_conf
 RunTest mesh_adv_proxy_mixin adv_tx_proxy_mixin adv_rx_proxy_mixin
 
+overlay=overlay_gatt_conf_overlay_workq_sys_conf
+RunTest mesh_adv_proxy_mixin_workq adv_tx_proxy_mixin adv_rx_proxy_mixin
+
 overlay="overlay_gatt_conf_overlay_psa_conf"
 RunTest mesh_adv_proxy_mixin_psa adv_tx_proxy_mixin adv_rx_proxy_mixin

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/random_order.sh

@@ -7,5 +7,8 @@ source $(dirname "${BASH_SOURCE[0]}")/../../_mesh_test.sh
 # test buffer management by filling buffers and sending them in random order.
 RunTest mesh_adv_random_order adv_tx_random_order adv_rx_random_order
 
+overlay=overlay_workq_sys_conf
+RunTest mesh_adv_random_order_workq adv_tx_random_order adv_rx_random_order
+
 overlay=overlay_psa_conf
 RunTest mesh_adv_random_order_psa adv_tx_random_order adv_rx_random_order

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/reverse_order.sh

@@ -7,5 +7,8 @@ source $(dirname "${BASH_SOURCE[0]}")/../../_mesh_test.sh
 # test buffer management by filling all the buffer and sending them in reversed order.
 RunTest mesh_adv_reverse_order adv_tx_reverse_order adv_rx_receive_order
 
+overlay=overlay_workq_sys_conf
+RunTest mesh_adv_reverse_order_workq adv_tx_reverse_order adv_rx_receive_order
+
 overlay=overlay_psa_conf
 RunTest mesh_adv_reverse_order_psa adv_tx_reverse_order adv_rx_receive_order

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/send_order.sh

@@ -7,5 +7,8 @@ source $(dirname "${BASH_SOURCE[0]}")/../../_mesh_test.sh
 # test buffer management by filling all the buffer and sending them all in order.
 RunTest mesh_adv_send_order adv_tx_send_order adv_rx_receive_order
 
+overlay=overlay_workq_sys_conf
+RunTest mesh_adv_send_order_workq adv_tx_send_order adv_rx_receive_order
+
 overlay=overlay_psa_conf
 RunTest mesh_adv_send_order_psa adv_tx_send_order adv_rx_receive_order

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/tx_cb_multi.sh

@@ -7,5 +7,8 @@ source $(dirname "${BASH_SOURCE[0]}")/../../_mesh_test.sh
 # test tx callbacks sequence for multiple advs
 RunTest mesh_adv_tx_cb_multi adv_tx_cb_multi
 
+overlay=overlay_workq_sys_conf
+RunTest mesh_adv_tx_cb_multi_workq adv_tx_cb_multi
+
 overlay=overlay_psa_conf
 RunTest mesh_adv_tx_cb_multi_psa adv_tx_cb_multi

tests/bsim/bluetooth/mesh/tests_scripts/advertiser/tx_cb_single.sh

@@ -7,5 +7,8 @@ source $(dirname "${BASH_SOURCE[0]}")/../../_mesh_test.sh
 # test tx callbacks parameters and xmit sequence for single adv
 RunTest mesh_adv_tx_cb_single adv_tx_cb_single adv_rx_xmit
 
+overlay=overlay_workq_sys_conf
+RunTest mesh_adv_tx_cb_single_workq adv_tx_cb_single adv_rx_xmit
+
 overlay=overlay_psa_conf
 RunTest mesh_adv_tx_cb_single_psa adv_tx_cb_single adv_rx_xmit

tests/bsim/bluetooth/mesh/tests_scripts/proxy_sol/sol_replay.sh

@@ -32,6 +32,18 @@ RunTest mesh_srpl_replay_attack \
 	proxy_sol_iut_power_replay_attack \
 	-flash=../results/mesh_srpl_replay_attack/flash.bin -flash_rm
 
+overlay="overlay_pst_conf_overlay_gatt_conf_overlay_workq_sys_conf"
+RunTest mesh_srpl_replay_attack_workq \
+	proxy_sol_tester_immediate_replay_attack \
+	proxy_sol_iut_immediate_replay_attack \
+	-flash=../results/mesh_srpl_replay_attack/flash.bin -flash_erase
+
+overlay="overlay_pst_conf_overlay_gatt_conf_overlay_workq_sys_conf"
+RunTest mesh_srpl_replay_attack_workq \
+	proxy_sol_tester_power_replay_attack \
+	proxy_sol_iut_power_replay_attack \
+	-flash=../results/mesh_srpl_replay_attack/flash.bin -flash_rm
+
 overlay="overlay_pst_conf_overlay_gatt_conf_overlay_psa_conf"
 RunTest mesh_srpl_replay_attack_psa \
 	proxy_sol_tester_immediate_replay_attack \