shell: Refactor command execution to enable raw arguments

Added special flag that can be used to indicate that optional
arguments are passed without any parsing (e.g. quotation marks
removal). Modified execute command to parse command line buffer
argument by argument.

After this change it is possible to forward whole command to
command handler (using select).

Signed-off-by: Krzysztof Chruscinski <krzysztof.chruscinski@nordicsemi.no>

include/shell/shell.h

@@ -36,6 +36,30 @@ extern "C" {
 
 #define SHELL_CMD_ROOT_LVL		(0u)
 
+/**
+ * @brief Flag indicates that optional arguments will be treated as one,
+ *	  unformatted argument.
+ *
+ * By default, shell is parsing all arguments, treats all spaces as argument
+ * separators unless they are within quotation marks which are removed in that
+ * case. If command rely on unformatted argument then this flag shall be used
+ * in place of number of optional arguments in command definition to indicate
+ * that only mandatory arguments shall be parsed and remaining command string is
+ * passed as a raw string.
+ */
+#define SHELL_OPT_ARG_RAW	(0xFE)
+
+/**
+ * @brief Flag indicateding that number of optional arguments is not limited.
+ */
+#define SHELL_OPT_ARG_CHECK_SKIP (0xFF)
+
+/**
+ * @brief Flag indicating maximum number of optional arguments that can be
+ *	  validated.
+ */
+#define SHELL_OPT_ARG_MAX		(0xFD)
+
 /**
  * @brief Shell API
  * @defgroup shell_api Shell API

subsys/shell/shell.c

@@ -218,7 +218,7 @@ static inline u16_t completion_space_get(const struct shell *shell)
 /* Prepare arguments and return number of space available for completion. */
 static bool tab_prepare(const struct shell *shell,
 			const struct shell_static_entry **cmd,
-			char ***argv, size_t *argc,
+			const char ***argv, size_t *argc,
 			size_t *complete_arg_idx,
 			struct shell_static_entry *d_entry)
 {
@@ -260,8 +260,8 @@ static bool tab_prepare(const struct shell *shell,
 
 	search_argc = space ? *argc : *argc - 1;
 
-	*cmd = shell_get_last_command(shell, search_argc, *argv,
+	*cmd = shell_get_last_command(shell->ctx->selected_cmd, search_argc,
-				      complete_arg_idx,	d_entry, false);
+				      *argv, complete_arg_idx,	d_entry, false);
 
 	/* if search_argc == 0 (empty command line) shell_get_last_command will
 	 * return NULL tab is allowed, otherwise not.
@@ -469,7 +469,7 @@ static void partial_autocomplete(const struct shell *shell,
 	}
 }
 
-static int exec_cmd(const struct shell *shell, size_t argc, char **argv,
+static int exec_cmd(const struct shell *shell, size_t argc, const char **argv,
 		    const struct shell_static_entry *help_entry)
 {
 	int ret_val = 0;
@@ -492,8 +492,10 @@ static int exec_cmd(const struct shell *shell, size_t argc, char **argv,
 	}
 
 	if (shell->ctx->active_cmd.args.mandatory) {
-		u8_t mand = shell->ctx->active_cmd.args.mandatory;
+		u32_t mand = shell->ctx->active_cmd.args.mandatory;
-		u8_t opt = shell->ctx->active_cmd.args.optional;
+		u8_t opt8 = shell->ctx->active_cmd.args.optional;
+		u32_t opt = (opt8 == SHELL_OPT_ARG_CHECK_SKIP) ?
+				UINT16_MAX : opt8;
 		bool in_range = (argc >= mand) && (argc <= (mand + opt));
 
 		/* Check if argc is within allowed range */
@@ -506,7 +508,8 @@ static int exec_cmd(const struct shell *shell, size_t argc, char **argv,
 		 */
 		k_mutex_unlock(&shell->ctx->wr_mtx);
 		flag_cmd_ctx_set(shell, 1);
-		ret_val = shell->ctx->active_cmd.handler(shell, argc, argv);
+		ret_val = shell->ctx->active_cmd.handler(shell, argc,
+							 (char **)argv);
 		flag_cmd_ctx_set(shell, 0);
 		/* Bring back mutex to shell thread. */
 		k_mutex_lock(&shell->ctx->wr_mtx, K_FOREVER);
@@ -515,23 +518,74 @@ static int exec_cmd(const struct shell *shell, size_t argc, char **argv,
 	return ret_val;
 }
 
+static void active_cmd_prepare(const struct shell_static_entry *entry,
+				struct shell_static_entry *active_cmd,
+				struct shell_static_entry *help_entry,
+				size_t *lvl, size_t *handler_lvl,
+				size_t *args_left)
+{
+	if (entry->handler) {
+		*handler_lvl = *lvl;
+		*active_cmd = *entry;
+		if ((entry->subcmd == NULL)
+		    && entry->args.optional == SHELL_OPT_ARG_RAW) {
+			*args_left = entry->args.mandatory - 1;
+			*lvl = *lvl + 1;
+		}
+	}
+	if (entry->help) {
+		*help_entry = *entry;
+	}
+}
+
+static bool wildcard_check_report(const struct shell *shell, bool found,
+				  const struct shell_static_entry *entry)
+{
+	/* An error occurred, fnmatch  argument cannot be followed by argument
+	 * with a handler to avoid multiple function calls.
+	 */
+	if (IS_ENABLED(CONFIG_SHELL_WILDCARD) && found && entry->handler) {
+		shell_op_cursor_end_move(shell);
+		shell_op_cond_next_line(shell);
+
+		shell_internal_fprintf(shell, SHELL_ERROR,
+			"Error: requested multiple function executions\n");
+		return false;
+	}
+
+	return true;
+}
+
 /* Function is analyzing the command buffer to find matching commands. Next, it
  * invokes the  last recognized command which has a handler and passes the rest
  * of command buffer as arguments.
+ *
+ * By default command buffer is parsed and spaces are treated by arguments
+ * separators. Complex arguments are provided in quotation marks with quotation
+ * marks escaped within the argument. Argument parser is removing quotation
+ * marks at argument boundary as well as escape characters within the argument.
+ * However, it is possible to indicate that command shall treat remaining part
+ * of command buffer as the last argument without parsing. This can be used for
+ * commands which expects whole command buffer to be passed directly to
+ * the command handler without any preprocessing.
+ * Because of that feature, command buffer is processed argument by argument and
+ * decision on further processing is based on currently processed command.
  */
 static int execute(const struct shell *shell)
 {
 	struct shell_static_entry dloc; /* Memory for dynamic commands. */
-	char *argv[CONFIG_SHELL_ARGC_MAX + 1]; /* +1 reserved for NULL */
+	const char *argv[CONFIG_SHELL_ARGC_MAX + 1]; /* +1 reserved for NULL */
 	const struct shell_static_entry *parent = shell->ctx->selected_cmd;
 	const struct shell_static_entry *entry = NULL;
 	struct shell_static_entry help_entry;
-	size_t cmd_lvl = SHELL_CMD_ROOT_LVL;
+	size_t cmd_lvl = 0;
 	size_t cmd_with_handler_lvl = 0;
 	bool wildcard_found = false;
-	size_t cmd_idx = 0;
+	size_t argc = 0, args_left = SIZE_MAX;
-	size_t argc;
 	char quote;
+	const char **argvp;
+	char *cmd_buf = shell->ctx->cmd_buff;
+	bool has_last_handler = false;
 
 	shell_op_cursor_end_move(shell);
 	if (!shell_cursor_in_empty_line(shell)) {
@@ -549,33 +603,37 @@ static int execute(const struct shell *shell)
 		shell_wildcard_prepare(shell);
 	}
 
-	/* create argument list */
+	/* Parent present means we are in select mode. */
-	quote = shell_make_argv(&argc, &argv[0], shell->ctx->cmd_buff,
+	if (parent != NULL) {
-				CONFIG_SHELL_ARGC_MAX);
+		argv[0] = parent->syntax;
-
+		argv[1] = cmd_buf;
-	if (!argc) {
+		argvp = &argv[1];
-		return -ENOEXEC;
+		active_cmd_prepare(parent, &shell->ctx->active_cmd, &help_entry,
+				   &cmd_lvl, &cmd_with_handler_lvl, &args_left);
+		cmd_lvl++;
+	} else {
+		help_entry.help = NULL;
+		argvp = &argv[0];
 	}
 
-	if (quote != 0) {
-		shell_internal_fprintf(shell, SHELL_ERROR,
-				       "not terminated: %c\n",
-				       quote);
-		return -ENOEXEC;
-	}
-
-	/* Initialize help variable */
-	help_entry.help = NULL;
-
 	/* Below loop is analyzing subcommands of found root command. */
-	while (true) {
+	while ((argc != 1) && (cmd_lvl <= CONFIG_SHELL_ARGC_MAX)
-		if (cmd_lvl >= argc) {
+		&& args_left > 0) {
-			break;
+		quote = shell_make_argv(&argc, argvp, cmd_buf, 2);
+		cmd_buf = (char *)argvp[1];
+
+		if (argc == 0) {
+			return -ENOEXEC;
+		} else if ((argc == 1) && (quote != 0)) {
+			shell_internal_fprintf(shell, SHELL_ERROR,
+					"not terminated: %c\n",
+					quote);
+			return -ENOEXEC;
 		}
 
 		if (IS_ENABLED(CONFIG_SHELL_HELP) && (cmd_lvl > 0) &&
-		    (!strcmp(argv[cmd_lvl], "-h") ||
+		    (!strcmp(argvp[0], "-h") ||
-		     !strcmp(argv[cmd_lvl], "--help"))) {
+		     !strcmp(argvp[0], "--help"))) {
 			/* Command called with help option so it makes no sense
 			 * to search deeper commands.
 			 */
@@ -594,7 +652,7 @@ static int execute(const struct shell *shell)
 			enum shell_wildcard_status status;
 
 			status = shell_wildcard_process(shell, entry,
-							argv[cmd_lvl]);
+							argvp[0]);
 			/* Wildcard character found but there is no matching
 			 * command.
 			 */
@@ -612,59 +670,33 @@ static int execute(const struct shell *shell)
 			}
 		}
 
-		entry = shell_cmd_get(parent, cmd_idx++, &dloc);
+		if (has_last_handler == false) {
+			entry = shell_find_cmd(parent, argvp[0], &dloc);
+		}
 
-		if ((cmd_idx == 0) || (entry == NULL)) {
+		argvp++;
-			if (cmd_lvl == 0 &&
+		args_left--;
-				(!shell_in_select_mode(shell) ||
+		if (entry) {
-				 shell->ctx->selected_cmd->handler == NULL)) {
+			if (wildcard_check_report(shell, wildcard_found, entry)
-				shell_internal_fprintf(shell, SHELL_ERROR,
+				== false) {
-						       "%s%s\n", argv[0],
-						       SHELL_MSG_CMD_NOT_FOUND);
 				return -ENOEXEC;
 			}
-			if (shell_in_select_mode(shell)		 &&
-			    shell->ctx->selected_cmd->handler != NULL) {
-				entry = shell->ctx->selected_cmd;
-				shell->ctx->active_cmd = *entry;
-				cmd_with_handler_lvl = cmd_lvl;
-			}
-			break;
-		}
 
-		if (strcmp(argv[cmd_lvl], entry->syntax) == 0) {
+			active_cmd_prepare(entry, &shell->ctx->active_cmd,
-			/* checking if command has a handler */
+					  &help_entry, &cmd_lvl,
-			if (entry->handler != NULL) {
+					  &cmd_with_handler_lvl, &args_left);
-				if (IS_ENABLED(CONFIG_SHELL_WILDCARD) &&
-				    (wildcard_found)) {
-					shell_op_cursor_end_move(shell);
-					shell_op_cond_next_line(shell);
-
-					/* An error occurred, fnmatch  argument
-					 * cannot be followed by argument with
-					 * a handler to avoid multiple function
-					 * calls.
-					 */
-					shell_internal_fprintf(shell,
-						SHELL_ERROR,
-						"Error: requested multiple"
-						" function executions\n");
-
-					return -ENOEXEC;
-				}
-
-				shell->ctx->active_cmd = *entry;
-				cmd_with_handler_lvl = cmd_lvl;
-			}
-			/* checking if function has a help handler */
-			if (entry->help != NULL) {
-				help_entry = *entry;
-			}
-
-			cmd_lvl++;
-			cmd_idx = 0;
 			parent = entry;
+		} else {
+			/* last handler found - no need to search commands in
+			 * the next iteration.
+			 */
+			has_last_handler = true;
 		}
+
+		if (args_left || (argc == 2)) {
+			cmd_lvl++;
+		}
+
 	}
 
 	if (IS_ENABLED(CONFIG_SHELL_WILDCARD) && wildcard_found) {
@@ -673,24 +705,32 @@ static int execute(const struct shell *shell)
 		 * with all expanded commands. Hence shell_make_argv needs to
 		 * be called again.
 		 */
-		(void)shell_make_argv(&argc, &argv[0],
+		(void)shell_make_argv(&cmd_lvl,
+				      &argv[shell->ctx->selected_cmd ? 1 : 0],
 				      shell->ctx->cmd_buff,
 				      CONFIG_SHELL_ARGC_MAX);
+
+		if (shell->ctx->selected_cmd) {
+			/* Apart from what is in the command buffer, there is
+			 * a selected command.
+			 */
+			cmd_lvl++;
+		}
 	}
 
 	/* Executing the deepest found handler. */
-	return exec_cmd(shell, argc - cmd_with_handler_lvl,
+	return exec_cmd(shell, cmd_lvl - cmd_with_handler_lvl,
 			&argv[cmd_with_handler_lvl], &help_entry);
 }
 
 static void tab_handle(const struct shell *shell)
 {
 	/* +1 reserved for NULL in function shell_make_argv */
-	char *__argv[CONFIG_SHELL_ARGC_MAX + 1];
+	const char *__argv[CONFIG_SHELL_ARGC_MAX + 1];
 	/* d_entry - placeholder for dynamic command */
 	struct shell_static_entry d_entry;
 	const struct shell_static_entry *cmd;
-	char **argv = __argv;
+	const char **argv = __argv;
 	size_t first = 0;
 	size_t arg_idx;
 	u16_t longest;
@@ -853,8 +893,7 @@ static void state_collect(const struct shell *shell)
 		switch (shell->ctx->receive_state) {
 		case SHELL_RECEIVE_DEFAULT:
 			if (process_nl(shell, data)) {
-				if (!shell->ctx->cmd_buff_len &&
+				if (!shell->ctx->cmd_buff_len) {
-				    shell->ctx->selected_cmd == NULL) {
 					history_mode_exit(shell);
 					cursor_next_line_move(shell);
 				} else {

subsys/shell/shell_cmds.c

@@ -393,8 +393,9 @@ static int cmd_select(const struct shell *shell, size_t argc, char **argv)
 
 	argc--;
 	argv = argv + 1;
-	candidate = shell_get_last_command(shell, argc, argv, &matching_argc,
+	candidate = shell_get_last_command(shell->ctx->selected_cmd,
-					   &entry, true);
+					   argc, (const char **)argv,
+					   &matching_argc, &entry, true);
 
 	if ((candidate != NULL) && !no_args(candidate)
 	    && (argc == matching_argc)) {
@@ -453,10 +454,12 @@ SHELL_STATIC_SUBCMD_SET_CREATE(m_sub_resize,
 
 SHELL_CMD_ARG_REGISTER(clear, NULL, SHELL_HELP_CLEAR, cmd_clear, 1, 0);
 SHELL_CMD_REGISTER(shell, &m_sub_shell, SHELL_HELP_SHELL, NULL);
-SHELL_CMD_ARG_REGISTER(help, NULL, SHELL_HELP_HELP, cmd_help, 1, 255);
+SHELL_CMD_ARG_REGISTER(help, NULL, SHELL_HELP_HELP, cmd_help,
+			1, SHELL_OPT_ARG_CHECK_SKIP);
 SHELL_COND_CMD_ARG_REGISTER(CONFIG_SHELL_HISTORY, history, NULL,
 			SHELL_HELP_HISTORY, cmd_history, 1, 0);
 SHELL_COND_CMD_ARG_REGISTER(CONFIG_SHELL_CMDS_RESIZE, resize, &m_sub_resize,
 			SHELL_HELP_RESIZE, cmd_resize, 1, 1);
 SHELL_COND_CMD_ARG_REGISTER(CONFIG_SHELL_CMDS_SELECT, select, NULL,
-			    SHELL_HELP_SELECT, cmd_select, 2, 255);
+			    SHELL_HELP_SELECT, cmd_select, 2,
+			    SHELL_OPT_ARG_CHECK_SKIP);

subsys/shell/shell_utils.c

@@ -12,7 +12,7 @@ extern const struct shell_cmd_entry __shell_root_cmds_end[0];
 
 static inline const struct shell_cmd_entry *shell_root_cmd_get(u32_t id)
 {
-	return  &__shell_root_cmds_start[id];
+	return &__shell_root_cmds_start[id];
 }
 
 /* Calculates relative line number of given position in buffer */
@@ -169,7 +169,7 @@ static char make_argv(char **ppcmd, u8_t c)
 }
 
 
-char shell_make_argv(size_t *argc, char **argv, char *cmd, u8_t max_argc)
+char shell_make_argv(size_t *argc, const char **argv, char *cmd, u8_t max_argc)
 {
 	char quote = 0;
 	char c;
@@ -187,8 +187,11 @@ char shell_make_argv(size_t *argc, char **argv, char *cmd, u8_t max_argc)
 		}
 
 		argv[(*argc)++] = cmd;
+		if (*argc == max_argc) {
+			break;
+		}
 		quote = make_argv(&cmd, c);
-	} while (*argc < max_argc);
+	} while (true);
 
 	argv[*argc] = 0;
 
@@ -272,7 +275,6 @@ const struct shell_static_entry *shell_cmd_get(
 
 /* Function returns pointer to a command matching given pattern.
  *
- * @param shell		Shell instance.
  * @param cmd		Pointer to commands array that will be searched.
  * @param lvl		Root command indicator. If set to 0 shell will search
  *			for pattern in parent commands.
@@ -281,10 +283,9 @@ const struct shell_static_entry *shell_cmd_get(
  *
  * @return		Pointer to found command.
  */
-static const struct shell_static_entry *find_cmd(
+const struct shell_static_entry *shell_find_cmd(
-					const struct shell *shell,
 					const struct shell_static_entry *parent,
-					char *cmd_str,
+					const char *cmd_str,
 					struct shell_static_entry *dloc)
 {
 	const struct shell_static_entry *entry;
@@ -300,15 +301,14 @@ static const struct shell_static_entry *find_cmd(
 }
 
 const struct shell_static_entry *shell_get_last_command(
-					     const struct shell *shell,
+					const struct shell_static_entry *entry,
-					     size_t argc,
+					size_t argc,
-					     char *argv[],
+					const char *argv[],
-					     size_t *match_arg,
+					size_t *match_arg,
-					     struct shell_static_entry *dloc,
+					struct shell_static_entry *dloc,
-					     bool only_static)
+					bool only_static)
 {
 	const struct shell_static_entry *prev_entry = NULL;
-	const struct shell_static_entry *entry = shell->ctx->selected_cmd;
 
 	*match_arg = SHELL_CMD_ROOT_LVL;
 
@@ -323,7 +323,7 @@ const struct shell_static_entry *shell_get_last_command(
 		}
 
 		prev_entry = entry;
-		entry = find_cmd(shell, entry, argv[*match_arg], dloc);
+		entry = shell_find_cmd(entry, argv[*match_arg], dloc);
 		if (entry) {
 			(*match_arg)++;
 		} else {

subsys/shell/shell_utils.h

@@ -36,7 +36,8 @@ static inline u16_t shell_strlen(const char *str)
 	return str == NULL ? 0U : (u16_t)strlen(str);
 }
 
-char shell_make_argv(size_t *argc, char **argv, char *cmd, uint8_t max_argc);
+char shell_make_argv(size_t *argc, const char **argv,
+		     char *cmd, uint8_t max_argc);
 
 /** @brief Removes pattern and following space
  *
@@ -56,11 +57,15 @@ const struct shell_static_entry *shell_cmd_get(
 					size_t idx,
 					struct shell_static_entry *dloc);
 
+const struct shell_static_entry *shell_find_cmd(
+					const struct shell_static_entry *parent,
+					const char *cmd_str,
+					struct shell_static_entry *dloc);
 
 /* @internal @brief Function returns pointer to a shell's subcommands array
  * for a level given by argc and matching command patter provided in argv.
  *
- * @param shell		Shell instance.
+ * @param shell		Entry. NULL for root entry.
  * @param argc		Number of arguments.
  * @param argv		Pointer to an array with arguments.
  * @param match_arg	Subcommand level of last matching argument.
@@ -70,12 +75,12 @@ const struct shell_static_entry *shell_cmd_get(
  * @return		Pointer to found command.
  */
 const struct shell_static_entry *shell_get_last_command(
-					     const struct shell *shell,
+					const struct shell_static_entry *entry,
-					     size_t argc,
+					size_t argc,
-					     char *argv[],
+					const char *argv[],
-					     size_t *match_arg,
+					size_t *match_arg,
-					     struct shell_static_entry *d_entry,
+					struct shell_static_entry *dloc,
-					     bool only_static);
+					bool only_static);
 
 int shell_command_add(char *buff, u16_t *buff_len,
 		      const char *new_cmd, const char *pattern);