shell: fix possible deadlock in shell_fprintf

Disable shell print functions if the shell is not initialized.
Update tests accordingly.

Fixes #27161

Signed-off-by: Jakub Rzeszutko <jakub.rzeszutko@nordisemi.no>
Signed-off-by: Steven Slupsky <sslupsky@gmail.com>

include/shell/shell_dummy.h

@@ -50,13 +50,20 @@ const struct shell *shell_backend_dummy_get_ptr(void);
  *
  * The returned data is always followed by a nul character at position *sizep
  *
- * @param shell		Shell pointer
+ * @param shell	Shell pointer
- * @param sizep		Returns size of data in shell buffer
+ * @param sizep	Returns size of data in shell buffer
  * @returns pointer to buffer containing shell output
  */
 const char *shell_backend_dummy_get_output(const struct shell *shell,
 					   size_t *sizep);
 
+/**
+ * @brief Clears the output buffer in the shell backend.
+ *
+ * @param shell	Shell pointer
+ */
+void shell_backend_dummy_clear_output(const struct shell *shell);
+
 #ifdef __cplusplus
 }
 #endif

subsys/shell/shell.c

@@ -1109,7 +1109,6 @@ static int instance_init(const struct shell *shell, const void *p_config,
 	flag_echo_set(shell, IS_ENABLED(CONFIG_SHELL_ECHO_STATUS));
 	flag_mode_delete_set(shell,
 			     IS_ENABLED(CONFIG_SHELL_BACKSPACE_MODE_DELETE));
-	shell->ctx->state = SHELL_STATE_INITIALIZED;
 	shell->ctx->vt100_ctx.cons.terminal_wid =
 					CONFIG_SHELL_DEFAULT_TERMINAL_WIDTH;
 	shell->ctx->vt100_ctx.cons.terminal_hei =
@@ -1117,9 +1116,14 @@ static int instance_init(const struct shell *shell, const void *p_config,
 	shell->ctx->vt100_ctx.cons.name_len = shell_strlen(shell->ctx->prompt);
 	flag_use_colors_set(shell, IS_ENABLED(CONFIG_SHELL_VT100_COLORS));
 
-	return shell->iface->api->init(shell->iface, p_config,
+	int ret = shell->iface->api->init(shell->iface, p_config,
-				       transport_evt_handler,
+					  transport_evt_handler,
-				       (void *) shell);
+					  (void *)shell);
+	if (ret == 0) {
+		shell->ctx->state = SHELL_STATE_INITIALIZED;
+	}
+
+	return ret;
 }
 
 static int instance_uninit(const struct shell *shell)
@@ -1348,6 +1352,11 @@ void shell_vfprintf(const struct shell *shell, enum shell_vt100_color color,
 	__ASSERT_NO_MSG(shell->fprintf_ctx);
 	__ASSERT_NO_MSG(fmt);
 
+	/* Sending a message to a non-active shell leads to a dead lock. */
+	if (shell->ctx->state != SHELL_STATE_ACTIVE) {
+		return;
+	}
+
 	k_mutex_lock(&shell->ctx->wr_mtx, K_FOREVER);
 	if (!flag_cmd_ctx_get(shell)) {
 		shell_cmd_line_erase(shell);

subsys/shell/shell_dummy.c

@@ -114,12 +114,19 @@ const struct shell *shell_backend_dummy_get_ptr(void)
 const char *shell_backend_dummy_get_output(const struct shell *shell,
 					   size_t *sizep)
 {
-	struct shell_dummy *sh_dummy;
+	struct shell_dummy *sh_dummy = (struct shell_dummy *)shell->iface->ctx;
 
-	sh_dummy = (struct shell_dummy *)shell->iface->ctx;
 	sh_dummy->buf[sh_dummy->len] = '\0';
 	*sizep = sh_dummy->len;
 	sh_dummy->len = 0;
 
 	return sh_dummy->buf;
 }
+
+void shell_backend_dummy_clear_output(const struct shell *shell)
+{
+	struct shell_dummy *sh_dummy = (struct shell_dummy *)shell->iface->ctx;
+
+	sh_dummy->buf[0] = '\0';
+	sh_dummy->len = 0;
+}

tests/shell/src/main.c

@@ -327,7 +327,7 @@ static void test_shell_fprintf(void)
 	zassert_not_null(shell, "Failed to get shell");
 
 	/* Clear the output buffer */
-	shell_backend_dummy_get_output(shell, &size);
+	shell_backend_dummy_clear_output(shell);
 
 	shell_fprintf(shell, SHELL_VT100_COLOR_DEFAULT, "testing %d %s %c",
 		      1, "2", '3');
@@ -389,5 +389,8 @@ void test_main(void)
 			ztest_unit_test(test_raw_arg)
 			);
 
+	/* Let the shell backend initialize. */
+	k_msleep(20);
+
 	ztest_run_test_suite(shell_test_suite);
 }