storage/stream: Add persistent write progress to stream_flash

Add additional API to stream_flash that can be used to make
stream write progress persistent using the settings subsystem.
This functionality makes it possible to resume a write operation
after it was interrupted, e.g. by power loss.

Signed-off-by: Jonathan Nilsen <Jonathan.Nilsen@nordicsemi.no>

doc/reference/storage/stream/stream_flash.rst

@@ -17,6 +17,17 @@ limitations which does not allow flash writes to be performed in parallell with
 other operations, such as radio RX and TX. Also, fewer write operations result
 in faster response times seen from the application.
 
+Persistent stream write progress
+********************************
+Some stream write operations, such as DFU operations, may run for a long time.
+When performing such long running operations it can be useful to be able to save
+the stream write progress to persistent storage so that the operation can resume
+at the same point after an unexpected interruption.
+
+The Stream Flash module offers an API for loading, saving and clearing stream
+write progress to persistent storage using the :ref:`Settings <settings_api>`
+module. The API can be enabled using :option:`CONFIG_STREAM_FLASH_PROGRESS`.
+
 API Reference
 *************
 

include/storage/stream_flash.h

@@ -128,6 +128,47 @@ int stream_flash_buffered_write(struct stream_flash_ctx *ctx, const uint8_t *dat
  */
 int stream_flash_erase_page(struct stream_flash_ctx *ctx, off_t off);
 
+/**
+ * @brief Load persistent stream write progress stored with key
+ *        @p settings_key .
+ *
+ * This function should be called directly after @ref stream_flash_init to
+ * load previous stream write progress before writing any data. If the loaded
+ * progress has fewer bytes written than @p ctx then it will be ignored.
+ *
+ * @param ctx context
+ * @param settings_key key to use with the settings module for loading
+ *                     the stream write progress
+ *
+ * @return non-negative on success, negative errno code on fail
+ */
+int stream_flash_progress_load(struct stream_flash_ctx *ctx,
+			       const char *settings_key);
+
+/**
+ * @brief Save persistent stream write progress using key @p settings_key .
+ *
+ * @param ctx context
+ * @param settings_key key to use with the settings module for storing
+ *                     the stream write progress
+ *
+ * @return non-negative on success, negative errno code on fail
+ */
+int stream_flash_progress_save(struct stream_flash_ctx *ctx,
+			       const char *settings_key);
+
+/**
+ * @brief Clear persistent stream write progress stored with key
+ *        @p settings_key .
+ *
+ * @param ctx context
+ * @param settings_key key previously used for storing the stream write progress
+ *
+ * @return non-negative on success, negative errno code on fail
+ */
+int stream_flash_progress_clear(struct stream_flash_ctx *ctx,
+				const char *settings_key);
+
 #ifdef __cplusplus
 }
 #endif

subsys/storage/stream/Kconfig

@@ -17,6 +17,15 @@ config STREAM_FLASH_ERASE
 	  If disabled an external actor must erase the flash area being written
 	  to.
 
+config STREAM_FLASH_PROGRESS
+	bool "Persistent stream write progress"
+	depends on SETTINGS
+	depends on !SETTINGS_NONE
+	help
+	  Enable API for loading and storing the current write progress to flash
+	  using the settings subsystem. In case of power failure or device
+	  reset, the API can be used to resume writing from the latest state.
+
 module = STREAM_FLASH
 module-str = stream flash
 source "subsys/logging/Kconfig.template.log_config"

subsys/storage/stream/stream_flash.c

@@ -16,6 +16,62 @@ LOG_MODULE_REGISTER(LOG_MODULE_NAME, CONFIG_STREAM_FLASH_LOG_LEVEL);
 
 #include <storage/stream_flash.h>
 
+#ifdef CONFIG_STREAM_FLASH_PROGRESS
+#include <settings/settings.h>
+
+static int settings_direct_loader(const char *key, size_t len,
+				  settings_read_cb read_cb, void *cb_arg,
+				  void *param)
+{
+	struct stream_flash_ctx *ctx = (struct stream_flash_ctx *) param;
+
+	/* Handle the subtree if it is an exact key match. */
+	if (settings_name_next(key, NULL) == 0) {
+		size_t bytes_written = 0;
+		ssize_t len = read_cb(cb_arg, &bytes_written,
+				      sizeof(bytes_written));
+
+		if (len != sizeof(ctx->bytes_written)) {
+			LOG_ERR("Unable to read bytes_written from storage");
+			return len;
+		}
+
+		/* Check that loaded progress is not outdated. */
+		if (bytes_written >= ctx->bytes_written) {
+			ctx->bytes_written = bytes_written;
+		} else {
+			LOG_WRN("Loaded outdated bytes_written %zu < %zu",
+				bytes_written, ctx->bytes_written);
+			return 0;
+		}
+
+#ifdef CONFIG_STREAM_FLASH_ERASE
+		int rc;
+		struct flash_pages_info page;
+		off_t offset = (off_t) (ctx->offset + ctx->bytes_written) - 1;
+
+		/* Update the last erased page to avoid deleting already
+		 * written data.
+		 */
+		if (ctx->bytes_written > 0) {
+			rc = flash_get_page_info_by_offs(ctx->fdev, offset,
+							 &page);
+			if (rc != 0) {
+				LOG_ERR("Error %d while getting page info", rc);
+				return rc;
+			}
+			ctx->last_erased_page_start_offset = page.start_offset;
+		} else {
+			ctx->last_erased_page_start_offset = -1;
+		}
+#endif /* CONFIG_STREAM_FLASH_ERASE */
+	}
+
+	return 0;
+}
+
+#endif /* CONFIG_STREAM_FLASH_PROGRESS */
+
 #ifdef CONFIG_STREAM_FLASH_ERASE
 
 int stream_flash_erase_page(struct stream_flash_ctx *ctx, off_t off)
@@ -201,6 +257,15 @@ int stream_flash_init(struct stream_flash_ctx *ctx, const struct device *fdev,
 		return -EFAULT;
 	}
 
+#ifdef CONFIG_STREAM_FLASH_PROGRESS
+	int rc = settings_subsys_init();
+
+	if (rc != 0) {
+		LOG_ERR("Error %d initializing settings subsystem", rc);
+		return rc;
+	}
+#endif
+
 	struct _inspect_flash inspect_flash_ctx = {
 		.buf_len = buf_len,
 		.total_size = 0
@@ -241,3 +306,62 @@ int stream_flash_init(struct stream_flash_ctx *ctx, const struct device *fdev,
 
 	return 0;
 }
+
+#ifdef CONFIG_STREAM_FLASH_PROGRESS
+
+int stream_flash_progress_load(struct stream_flash_ctx *ctx,
+			       const char *settings_key)
+{
+	if (!ctx || !settings_key) {
+		return -EFAULT;
+	}
+
+	int rc = settings_load_subtree_direct(settings_key,
+					      settings_direct_loader,
+					      (void *) ctx);
+
+	if (rc != 0) {
+		LOG_ERR("Error %d while loading progress for \"%s\"",
+			rc, settings_key);
+	}
+
+	return rc;
+}
+
+int stream_flash_progress_save(struct stream_flash_ctx *ctx,
+			       const char *settings_key)
+{
+	if (!ctx || !settings_key) {
+		return -EFAULT;
+	}
+
+	int rc = settings_save_one(settings_key,
+				   &ctx->bytes_written,
+				   sizeof(ctx->bytes_written));
+
+	if (rc != 0) {
+		LOG_ERR("Error %d while storing progress for \"%s\"",
+			rc, settings_key);
+	}
+
+	return rc;
+}
+
+int stream_flash_progress_clear(struct stream_flash_ctx *ctx,
+				const char *settings_key)
+{
+	if (!ctx || !settings_key) {
+		return -EFAULT;
+	}
+
+	int rc = settings_delete(settings_key);
+
+	if (rc != 0) {
+		LOG_ERR("Error %d while deleting progress for \"%s\"",
+			rc, settings_key);
+	}
+
+	return rc;
+}
+
+#endif  /* CONFIG_STREAM_FLASH_PROGRESS */

tests/subsys/storage/stream/stream_flash/prj.conf

@@ -7,7 +7,11 @@
 CONFIG_ZTEST=y
 CONFIG_FLASH=y
 CONFIG_FLASH_PAGE_LAYOUT=y
+CONFIG_FLASH_MAP=y
+CONFIG_NVS=y
+CONFIG_SETTINGS=y
 CONFIG_DEBUG_OPTIMIZATIONS=y
 
 CONFIG_STREAM_FLASH=y
 CONFIG_STREAM_FLASH_ERASE=y
+CONFIG_STREAM_FLASH_PROGRESS=y

tests/subsys/storage/stream/stream_flash/src/main.c

@@ -9,6 +9,7 @@
 #include <stdbool.h>
 #include <ztest.h>
 #include <drivers/flash.h>
+#include <settings/settings.h>
 
 #include <storage/stream_flash.h>
 
@@ -35,6 +36,8 @@ static size_t cb_len;
 static size_t cb_offset;
 static int cb_ret;
 
+static const char progress_key[] = "sf-test/progress";
+
 static uint8_t buf[BUF_LEN];
 static uint8_t read_buf[TESTBUF_SIZE];
 const static uint8_t write_buf[TESTBUF_SIZE] = {[0 ... TESTBUF_SIZE - 1] = 0xaa};
@@ -456,6 +459,186 @@ static void test_stream_flash_buffered_write_whole_page(void)
 }
 #endif
 
+static size_t write_and_save_progress(size_t bytes, const char *save_key)
+{
+	int rc;
+	size_t bytes_written;
+
+	rc = stream_flash_buffered_write(&ctx, write_buf, bytes, true);
+	zassert_equal(rc, 0, "expected success");
+
+	bytes_written = stream_flash_bytes_written(&ctx);
+	zassert_true(bytes_written > 0, "expected bytes to be written");
+
+	if (save_key) {
+		rc = stream_flash_progress_save(&ctx, save_key);
+		zassert_equal(rc, 0, "expected success");
+	}
+
+	return bytes_written;
+}
+
+static void clear_all_progress(void)
+{
+	(void) settings_delete(progress_key);
+}
+
+static size_t load_progress(const char *load_key)
+{
+	int rc;
+
+	rc = stream_flash_progress_load(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	return stream_flash_bytes_written(&ctx);
+}
+
+static void test_stream_flash_progress_api(void)
+{
+	int rc;
+
+	clear_all_progress();
+	init_target();
+
+	/* Test save parameter validation */
+	rc = stream_flash_progress_save(NULL, progress_key);
+	zassert_true(rc < 0, "expected error since ctx is NULL");
+
+	rc = stream_flash_progress_save(&ctx, NULL);
+	zassert_true(rc < 0, "expected error since key is NULL");
+
+	rc = stream_flash_progress_save(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	(void) write_and_save_progress(BUF_LEN, progress_key);
+
+	/* Test load parameter validation */
+	rc = stream_flash_progress_load(NULL, progress_key);
+	zassert_true(rc < 0, "expected error since ctx is NULL");
+
+	rc = stream_flash_progress_load(&ctx, NULL);
+	zassert_true(rc < 0, "expected error since key is NULL");
+
+	rc = stream_flash_progress_load(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	/* Test clear parameter validation */
+	rc = stream_flash_progress_clear(NULL, progress_key);
+	zassert_true(rc < 0, "expected error since ctx is NULL");
+
+	rc = stream_flash_progress_clear(&ctx, NULL);
+	zassert_true(rc < 0, "expected error since key is NULL");
+
+	rc = stream_flash_progress_clear(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+}
+
+static void test_stream_flash_progress_resume(void)
+{
+	int rc;
+	size_t bytes_written_old;
+	size_t bytes_written;
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	off_t erase_offset_old;
+	off_t erase_offset;
+#endif
+
+	clear_all_progress();
+	init_target();
+
+	bytes_written_old = stream_flash_bytes_written(&ctx);
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	erase_offset_old = ctx.last_erased_page_start_offset;
+#endif
+
+	/* Test load with zero bytes_written */
+	rc = stream_flash_progress_save(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	rc = stream_flash_progress_load(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	bytes_written = stream_flash_bytes_written(&ctx);
+	zassert_equal(bytes_written, bytes_written_old,
+		      "expected bytes_written to be unchanged");
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	erase_offset = ctx.last_erased_page_start_offset;
+	zassert_equal(erase_offset, erase_offset_old,
+		      "expected erase offset to be unchanged");
+#endif
+
+	clear_all_progress();
+	init_target();
+
+	/* Write some data and save the progress */
+	bytes_written_old = write_and_save_progress(page_size * 2,
+						    progress_key);
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	erase_offset_old = ctx.last_erased_page_start_offset;
+	zassert_true(erase_offset_old != 0, "expected pages to be erased");
+#endif
+
+	init_target();
+
+	/* Load the previous progress */
+	bytes_written = load_progress(progress_key);
+	zassert_equal(bytes_written, bytes_written_old,
+		      "expected bytes_written to be loaded");
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	zassert_equal(erase_offset_old, ctx.last_erased_page_start_offset,
+		      "expected last erased page offset to be loaded");
+#endif
+
+	/* Check that outdated progress does not overwrite current progress */
+	init_target();
+
+	(void) write_and_save_progress(BUF_LEN, progress_key);
+	bytes_written_old = write_and_save_progress(BUF_LEN, NULL);
+	bytes_written = load_progress(progress_key);
+	zassert_equal(bytes_written, bytes_written_old,
+		      "expected bytes_written to not be overwritten");
+}
+
+static void test_stream_flash_progress_clear(void)
+{
+	int rc;
+	size_t bytes_written_old;
+	size_t bytes_written;
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	off_t erase_offset_old;
+	off_t erase_offset;
+#endif
+
+	clear_all_progress();
+	init_target();
+
+	/* Test that progress is cleared. */
+	(void) write_and_save_progress(BUF_LEN, progress_key);
+
+	rc = stream_flash_progress_clear(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	init_target();
+
+	bytes_written_old = stream_flash_bytes_written(&ctx);
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	erase_offset_old = ctx.last_erased_page_start_offset;
+#endif
+
+	rc = stream_flash_progress_load(&ctx, progress_key);
+	zassert_equal(rc, 0, "expected success");
+
+	bytes_written = stream_flash_bytes_written(&ctx);
+	zassert_equal(bytes_written, bytes_written_old,
+		      "expected bytes_written to be unchanged");
+
+#ifdef CONFIG_STREAM_FLASH_ERASE
+	erase_offset = ctx.last_erased_page_start_offset;
+	zassert_equal(erase_offset, erase_offset_old,
+		      "expected erase offset to be unchanged");
+#endif
+}
+
 void test_main(void)
 {
 	fdev = device_get_binding(FLASH_NAME);
@@ -476,7 +659,10 @@ void test_main(void)
 	     ztest_unit_test(test_stream_flash_flush),
 	     ztest_unit_test(test_stream_flash_buffered_write_whole_page),
 	     ztest_unit_test(test_stream_flash_erase_page),
-	     ztest_unit_test(test_stream_flash_bytes_written)
+	     ztest_unit_test(test_stream_flash_bytes_written),
+	     ztest_unit_test(test_stream_flash_progress_api),
+	     ztest_unit_test(test_stream_flash_progress_resume),
+	     ztest_unit_test(test_stream_flash_progress_clear)
 	 );
 
 	ztest_run_test_suite(lib_stream_flash_test);