native: added stdin handling for shell

Now the native console driver also handles stdin
so we can drive the shell from the command line,
a pipe or a file

Signed-off-by: Alberto Escolar Piedras <alpi@oticon.com>

boards/posix/native_posix/Kconfig.board

@@ -3,6 +3,7 @@ config BOARD_NATIVE_POSIX
 	bool "Native POSIX"
 	depends on SOC_POSIX
 	select NATIVE_POSIX_TIMER
+	select NATIVE_POSIX_CONSOLE
 	help
 	  Will produce a console Linux process which can be executed natively.
 	  It provides some minimal needed models:

boards/posix/native_posix/doc/board.rst

@@ -342,16 +342,23 @@ Peripherals
 
 The following peripherals are currently provided with this board:
 
-**Console/printk driver**:
+**Console driver**:
-  A driver is provided that redirects any printk write to the native
+  A console driver is provided which by default is configured to:
-  host application's stdout.
+
+  - Redirect any :c:func:`printk` write to the native host application's
+    `stdout`.
+
+  - Feed any input from the native application `stdin` to a possible
+    running :ref:`Shell`. For more information refer to the section
+    `Shell support`_.
 
 **Simple timer**:
   A simple timer provides the kernel with a 10ms tick.
   This peripheral driver also provides the needed functionality for this
   architecture-specific :c:func:`k_busy_wait`.
 
-  This timer, is configured by default with NATIVE_POSIX_SLOWDOWN_TO_REAL_TIME_
+  This timer, is configured by default with
+  :option:`CONFIG_NATIVE_POSIX_SLOWDOWN_TO_REAL_TIME`
   to slow down the execution to real host time.
   This will provide the illusion that the simulated time is running at the same
   speed as the real host time.
@@ -361,11 +368,66 @@ The following peripherals are currently provided with this board:
   Normally the Zephyr application and HW models run in very little time
   on the host CPU, so this is a good enough approach.
 
-.. _NATIVE_POSIX_SLOWDOWN_TO_REAL_TIME:
-   ../../../../reference/kconfig/CONFIG_NATIVE_POSIX_SLOWDOWN_TO_REAL_TIME.html
-
 **Interrupt controller**:
   A simple yet generic interrupt controller is provided. It can nest interrupts
   and provides interrupt priorities. Interrupts can be individually masked or
   unmasked. SW interrupts are also supported.
 
+Shell support
+*************
+
+When the :ref:`Shell` subsystem is compiled with your application, the native
+standard input (`stdin`) will be redirected to the shell.
+You may use the shell interactively through the console,
+by piping another process output to it, or by feeding it a file.
+
+When using it interactively you may want to select the option
+:option:`CONFIG_NATIVE_POSIX_SLOWDOWN_TO_REAL_TIME`.
+
+When feeding `stdin` from a pipe or file, the console driver will ensure
+reproducibility between runs of the process:
+
+- The execution of the process will be stalled while waiting for new `stdin`
+  data to be ready.
+
+- Commands will be fed to the shell as fast as the shell can process them.
+  To allow controlling the flow of commands to the shell, you may use the
+  driver directive ``!wait <ms>``.
+
+- When the file ends, or the pipe is closed the driver will stop attempting to
+  read it.
+
+Driver directives
+=================
+
+The console driver understands a set of special commands: driver directives.
+These directives are captured by the console driver itself and are not
+forwarded to the shell.
+These directives are:
+
+- ``!wait <ms>``: When received, the driver will pause feeding commands to the
+  shell for `<ms>` milliseconds.
+
+- ``!quit``: When received the driver will cause the application to gracefully
+  exit by calling :c:func:`posix_exit`.
+
+
+Use example
+===========
+
+For example, you can build the shell sample app:
+
+.. zephyr-app-commands::
+   :zephyr-app: samples/subsys/shell/shell_module
+   :host-os: unix
+   :board: native_posix
+   :goals: build
+   :compact:
+
+And feed it the following set of commands through a pipe:
+
+.. code-block:: console
+
+   echo -e \
+   'select kernel\nuptime\n!wait 500\nuptime\n!wait 1000\nuptime\n!quit' \
+   | zephyr/zephyr.exe

drivers/console/Kconfig

@@ -200,12 +200,44 @@ config XTENSA_SIM_CONSOLE
 
 config NATIVE_POSIX_CONSOLE
 	bool
-	prompt "Print to console"
+	prompt "Use native stdin/stdout for console"
-	depends on BOARD_NATIVE_POSIX
+	depends on ARCH_POSIX
 	select CONSOLE_HAS_DRIVER
 	default y
 	help
-	  Use native console to print messages.
+	  Use native stdin and stdout to print messages and get input
+
+config NATIVE_POSIX_STDIN_CONSOLE
+	bool "Use native stdin for console"
+	prompt "Read from native stdin for console"
+	depends on NATIVE_POSIX_CONSOLE
+	default y
+	help
+	  Use native console to get input.
+
+config NATIVE_STDIN_POLL_PERIOD
+	int
+	prompt "Polling period for stdin"
+	depends on NATIVE_POSIX_STDIN_CONSOLE
+	default 20
+	help
+	  In ms, polling period for stdin
+
+config NATIVE_STDIN_PRIO
+	int
+	prompt "Priority of the stdin polling thread"
+	depends on NATIVE_POSIX_STDIN_CONSOLE
+	default 4
+	help
+	  Prioriry of the native stdin polling thread
+
+config NATIVE_POSIX_STDOUT_CONSOLE
+	bool "Use native stdout for console"
+	prompt "Print to native stdout"
+	depends on NATIVE_POSIX_CONSOLE
+	default y
+	help
+	  Use native console (stout) to print messages.
 
 config XTENSA_CONSOLE_INIT_PRIORITY
 	int

drivers/console/native_posix_console.c

@@ -1,22 +1,35 @@
 /*
- * Copyright (c) 2017 Oticon A/S
+ * Copyright (c) 2018 Oticon A/S
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
 #include <stdio.h>
+#include <ctype.h>
 #include "init.h"
+#include "kernel.h"
+#include "console/console.h"
+#include <string.h>
+#include <sys/time.h>
+#include <sys/select.h>
+#include <unistd.h>
 
+#define DEBUG_ECHO 0
+
+#if (DEBUG_ECHO)
+#define ECHO(...) printf(__VA_ARGS__)
+#else
+#define ECHO(...)
+#endif
+
+#if defined(CONFIG_NATIVE_POSIX_STDOUT_CONSOLE)
 /**
  *
  * @brief Initialize the driver that provides the printk output
  *
- * @return 0 if successful, otherwise failed.
  */
-static int printk_init(struct device *arg)
+static void native_posix_stdout_init(void)
 {
-	ARG_UNUSED(arg);
-
 	/* Let's ensure that even if we are redirecting to a file, we get stdout
 	 * and stderr line buffered (default for console). Note that glibc
 	 * ignores size. But just in case we set a reasonable number in case
@@ -27,9 +40,245 @@ static int printk_init(struct device *arg)
 
 	extern void __printk_hook_install(int (*fn)(int));
 	__printk_hook_install(putchar);
+}
+
+/**
+ * Ensure that whatever was written thru printk is displayed now
+ */
+void posix_flush_stdout(void)
+{
+	fflush(stdout);
+}
+#endif /* CONFIG_NATIVE_POSIX_STDOUT_CONSOLE */
+
+#if defined(CONFIG_NATIVE_POSIX_STDIN_CONSOLE)
+
+#define VALID_DIRECTIVES \
+	"Valid native console driver directives:\n" \
+	"  !wait %%u\n" \
+	"  !quit\n"
+
+static struct k_fifo *avail_queue;
+static struct k_fifo *lines_queue;
+static u8_t (*completion_cb)(char *line, u8_t len);
+static bool stdin_is_tty;
+
+static K_THREAD_STACK_DEFINE(stack, CONFIG_ARCH_POSIX_RECOMMENDED_STACK_SIZE);
+static struct k_thread native_stdio_thread;
+
+static inline void found_eof(void)
+{
+	/*
+	 * Once stdin is closed or the input file has ended,
+	 * there is no need to try again
+	 */
+	ECHO("Got EOF\n");
+	k_thread_abort(&native_stdio_thread);
+}
+
+/*
+ * Check if the command is a directive the driver handles on its own
+ * and if it is, handle it.
+ * If not return 0 (so it can be passed to the shell)
+ *
+ * Inputs
+ *   s       Command string
+ *   towait  Pointer to the amount of time wait until attempting to receive
+ *           the next command
+ *
+ * return 0 if it is not a directive
+ * return > 0 if it was a directive (command starts with '!')
+ * return 2 if the driver directive requires to pause processing input
+ */
+static int catch_directive(char *s, s32_t *towait)
+{
+	while (*s != 0  && isspace(*s)) {
+		s++;
+	}
+
+	if (*s != '!') {
+		return 0;
+	}
+
+	if (strncmp(s, "!wait", 5) == 0) {
+		int ret;
+
+		ret = sscanf(&s[5], "%i", towait);
+		if (ret != 1) {
+			posix_print_error_and_exit("%s(): '%s' not understood, "
+						   "!wait syntax: !wait %%i\n",
+						   __func__, s);
+		}
+		return 2;
+	} else if (strcmp(s, "!quit") == 0) {
+		posix_exit(0);
+	}
+
+	posix_print_warning("%s(): '%s' not understood\n" VALID_DIRECTIVES,
+			    __func__, s);
+
+	return 1;
+}
+
+/**
+ * Check if there is data ready in stdin
+ */
+static int stdin_not_ready(void)
+{
+	int ready;
+	fd_set readfds;
+	struct timeval timeout;
+
+	timeout.tv_usec = 0;
+	timeout.tv_sec  = 0;
+
+	FD_ZERO(&readfds);
+	FD_SET(STDIN_FILENO, &readfds);
+
+	ready = select(STDIN_FILENO+1, &readfds, NULL, NULL, &timeout);
+
+	if (ready == 0) {
+		return 1;
+	} else if (ready == -1) {
+		posix_print_error_and_exit("%s: Error on select ()\n",
+					   __func__);
+	}
 
 	return 0;
 }
 
-SYS_INIT(printk_init, PRE_KERNEL_1, CONFIG_NATIVE_POSIX_CONSOLE_INIT_PRIORITY);
+/**
+ * Check if there is any line in the stdin buffer,
+ * if there is and we have available shell buffers feed it to the shell
+ *
+ * This function returns how long the thread should wait in ms,
+ * before checking again the stdin buffer
+ */
+static s32_t attempt_read_from_stdin(void)
+{
+	static struct console_input *cmd;
+	s32_t towait = CONFIG_NATIVE_STDIN_POLL_PERIOD;
+
+	while (1) {
+		char *ret;
+		int last;
+		int is_directive;
+
+		if (feof(stdin)) {
+			found_eof();
+		}
+
+		/*
+		 * If stdin comes from a terminal, we check if the user has
+		 * input something, and if not we pause the process.
+		 *
+		 * If stdin is not coming from a terminal, but from a file or
+		 * pipe, we always proceed to try to get data and block until
+		 * we do
+		 */
+		if (stdin_is_tty && stdin_not_ready()) {
+			return towait;
+		}
+
+		/* Pick next available shell line buffer */
+		if (!cmd) {
+			cmd = k_fifo_get(avail_queue, K_NO_WAIT);
+			if (!cmd) {
+				return towait;
+			}
+		}
+
+		/*
+		 * By default stdin is (_IOLBF) line buffered when connected to
+		 * a terminal and fully buffered (_IOFBF) when connected to a
+		 * pipe/file.
+		 * If we got a terminal: we already checked for it to be ready
+		 * and therefore a full line should be there for us.
+		 *
+		 * If we got a pipe or file we will block until we get a line,
+		 * or we reach EOF
+		 */
+		ret = fgets(cmd->line, CONSOLE_MAX_LINE_LEN, stdin);
+		if (ret == NULL) {
+			if (feof(stdin)) {
+				found_eof();
+			}
+			/*
+			 * Otherwise this was an unexpected error we do
+			 * not try to handle
+			 */
+			return towait;
+		}
+
+		/* Remove a possible end of line and other trailing spaces */
+		last = (int)strlen(cmd->line) - 1;
+		while ((last >= 0) && isspace(cmd->line[last])) {
+			cmd->line[last--] = 0;
+		}
+
+		ECHO("Got: \"%s\"\n", cmd->line);
+
+		/*
+		 * This console has a special set of directives which start with
+		 * "!" which we capture here
+		 */
+		is_directive = catch_directive(cmd->line, &towait);
+		if (is_directive == 2) {
+			return towait;
+		} else if (is_directive > 0) {
+			continue;
+		}
+
+		/* Let's give it to the shell to handle */
+		k_fifo_put(lines_queue, cmd);
+		cmd = NULL;
+	}
+
+	return towait;
+}
+
+/**
+ * This thread will check if there is any new line in the stdin buffer
+ * every CONFIG_NATIVE_STDIN_POLL_PERIOD ms
+ *
+ * If there is, it will feed it to the shell
+ */
+static void native_stdio_runner(void *p1, void *p2, void *p3)
+{
+	stdin_is_tty = isatty(STDIN_FILENO);
+
+	while (1) {
+		s32_t wait_time = attempt_read_from_stdin();
+
+		k_sleep(wait_time);
+	}
+}
+
+void native_stdin_register_input(struct k_fifo *avail, struct k_fifo *lines,
+			 u8_t (*completion)(char *str, u8_t len))
+{
+	avail_queue = avail;
+	lines_queue = lines;
+	completion_cb = completion;
+
+	k_thread_create(&native_stdio_thread, stack,
+			CONFIG_ARCH_POSIX_RECOMMENDED_STACK_SIZE,
+			native_stdio_runner, NULL, NULL, NULL,
+			CONFIG_NATIVE_STDIN_PRIO, 0, K_NO_WAIT);
+}
+#endif /* CONFIG_NATIVE_POSIX_STDIN_CONSOLE */
+
+static int native_posix_console_init(struct device *arg)
+{
+	ARG_UNUSED(arg);
+
+#if defined(CONFIG_NATIVE_POSIX_STDOUT_CONSOLE)
+	native_posix_stdout_init();
+#endif
+
+	return 0;
+}
+
+SYS_INIT(native_posix_console_init, PRE_KERNEL_1,
+	CONFIG_NATIVE_POSIX_CONSOLE_INIT_PRIORITY);
 

include/drivers/console/native_posix_console.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018 Oticon A/S
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef _NATIVE_POSIX_CONSOLE_H
+#define _NATIVE_POSIX_CONSOLE_H
+
+#include "kernel.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(CONFIG_NATIVE_POSIX_STDOUT_CONSOLE)
+void posix_flush_stdout(void);
+#endif
+
+#if defined(CONFIG_NATIVE_POSIX_STDIN_CONSOLE)
+void native_stdin_register_input(struct k_fifo *avail, struct k_fifo *lines,
+			 u8_t (*completion)(char *str, u8_t len));
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _NATIVE_POSIX_CONSOLE_H */

subsys/shell/shell.c

@@ -25,6 +25,9 @@
 #ifdef CONFIG_TELNET_CONSOLE
 #include <console/telnet_console.h>
 #endif
+#ifdef CONFIG_NATIVE_POSIX_CONSOLE
+#include <console/native_posix_console.h>
+#endif
 
 #include <shell/shell.h>
 
@@ -422,6 +425,10 @@ static void shell(void *p1, void *p2, void *p3)
 		struct console_input *cmd;
 
 		printk("%s", get_prompt());
+#if defined(CONFIG_NATIVE_POSIX_CONSOLE)
+		/* The native printk driver is line buffered */
+		posix_flush_stdout();
+#endif
 
 		cmd = k_fifo_get(&cmds_queue, K_FOREVER);
 
@@ -595,6 +602,9 @@ void shell_init(const char *str)
 #ifdef CONFIG_TELNET_CONSOLE
 	telnet_register_input(&avail_queue, &cmds_queue, completion);
 #endif
+#ifdef CONFIG_NATIVE_POSIX_STDIN_CONSOLE
+	native_stdin_register_input(&avail_queue, &cmds_queue, completion);
+#endif
 }
 
 /** @brief Optionally register an app default cmd handler.