syscall_handler: handle multiplication overflow

Computing the total size of the array need to handle the case where
the product overflow a 32-bit unsigned integer.

Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>

kernel/include/syscall_handler.h

@@ -116,6 +116,47 @@ int _k_object_validate(void *obj, enum k_objects otype, int init);
 #define _SYSCALL_MEMORY_WRITE(ptr, size, ssf) \
 	_SYSCALL_MEMORY(ptr, size, 1, ssf)
 
+#define _SYSCALL_MEMORY_ARRAY(ptr, nmemb, size, write, ssf) \
+	do { \
+		u32_t product; \
+		_SYSCALL_VERIFY_MSG(!__builtin_umul_overflow((u32_t)(nmemb), \
+							     (u32_t)(size), \
+							     &product), ssf, \
+				    "%ux%u array is too large", \
+				    (u32_t)(nmemb), (u32_t)(size)); \
+		_SYSCALL_MEMORY(ptr, product, write, ssf); \
+	} while (0)
+
+/**
+ * @brief Validate user thread has read permission for sized array
+ *
+ * Used when the memory region is expressed in terms of number of elements and
+ * each element size, handles any overflow issues with computing the total
+ * array bounds. Otherwise see _SYSCALL_MEMORY_READ.
+ *
+ * @param ptr Memory area to examine
+ * @param nmemb Number of elements in the array
+ * @param size Size of each array element
+ * @param ssf Syscall stack frame argument passed to the handler function
+ */
+#define _SYSCALL_MEMORY_ARRAY_READ(ptr, nmemb, size, ssf) \
+	_SYSCALL_MEMORY_ARRAY(ptr, nmemb, size, 0, ssf)
+
+/**
+ * @brief Validate user thread has read/write permission for sized array
+ *
+ * Used when the memory region is expressed in terms of number of elements and
+ * each element size, handles any overflow issues with computing the total
+ * array bounds. Otherwise see _SYSCALL_MEMORY_WRITE.
+ *
+ * @param ptr Memory area to examine
+ * @param nmemb Number of elements in the array
+ * @param size Size of each array element
+ * @param ssf Syscall stack frame argument passed to the handler function
+ */
+#define _SYSCALL_MEMORY_ARRAY_WRITE(ptr, nmemb, size, ssf) \
+	_SYSCALL_MEMORY_ARRAY(ptr, nmemb, size, 1, ssf)
+
 #define _SYSCALL_IS_OBJ(ptr, type, init, ssf) \
 	_SYSCALL_VERIFY_MSG(!_k_object_validate((void *)ptr, type, init), ssf, \
 			    "object %p access denied", (void *)(ptr))

kernel/msg_q.c

@@ -70,7 +70,7 @@ u32_t _handler_k_msgq_init(u32_t q, u32_t buffer, u32_t msg_size,
 	_SYSCALL_ARG4;
 
 	_SYSCALL_OBJ_INIT(q, K_OBJ_MSGQ, ssf);
-	_SYSCALL_MEMORY_WRITE(buffer, msg_size * max_msgs, ssf);
+	_SYSCALL_MEMORY_ARRAY_WRITE(buffer, max_msgs, msg_size, ssf);
 
 	_impl_k_msgq_init((struct k_msgq *)q, (char *)buffer, msg_size,
 			  max_msgs);

kernel/stack.c

@@ -64,7 +64,7 @@ u32_t _handler_k_stack_init(u32_t stack, u32_t buffer, u32_t num_entries_p,
 	/* FIXME why is 'num_entries' signed?? */
 	_SYSCALL_VERIFY(num_entries > 0, ssf);
 	_SYSCALL_OBJ_INIT(stack, K_OBJ_STACK, ssf);
-	_SYSCALL_MEMORY_WRITE(buffer, num_entries * sizeof(u32_t), ssf);
+	_SYSCALL_MEMORY_ARRAY_WRITE(buffer, num_entries, sizeof(u32_t), ssf);
 
 	_impl_k_stack_init((struct k_stack *)stack, (u32_t *)buffer,
 			   num_entries);