michaelh/zephyr
1
0
Fork 0
Code Releases Activity

samples: add tensorflow magic wand sample

Browse source 
Adds Tensorflow Magic Wand sample to tree.

Signed-off-by: Lauren Murphy <lauren.murphy@intel.com>
This commit is contained in:
Lauren Murphy 2021-04-26 22:07:04 -05:00 • committed by Anas Nashif
commit 83a036d738
24 changed files with 2920 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

24
samples/modules/tensorflow/magic_wand/CMakeLists.txt Normal file
View file

@ -0,0 +1,24 @@
cmake_minimum_required(VERSION 3.13.1)
find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
project(tensorflow_magic_wand)
# Required for TensorFlow to compile properly
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-threadsafe-statics")
target_sources(app PRIVATE
src/main_functions.h
src/constants.h
src/magic_wand_model_data.h
src/gesture_predictor.h
src/output_handler.h
src/assert.cc
src/accelerometer_handler.cc
src/accelerometer_handler.h
src/main.cc
src/main_functions.cc
src/magic_wand_model_data.cc
src/gesture_predictor.cc
src/output_handler.cc
boards/litex_vexriscv.overlay
)

125
samples/modules/tensorflow/magic_wand/README.rst Normal file
View file

@ -0,0 +1,125 @@
.. _tensorflow_magic_wand:
TensorFlow Magic Wand sample
############################
Overview
********
This sample application shows how to use TensorFlow Lite Micro
to run a 20 kilobyte neural network model that recognizes gestures
from an accelerometer.
.. Note::
This README and sample have been modified from
`the TensorFlow Magic Wand sample for Zephyr`_ and
`the Antmicro tutorial on Renode emulation for TensorFlow`_.
.. _the TensorFlow Magic Wand sample for Zephyr:
https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/examples/magic_wand
.. _the Antmicro tutorial on Renode emulation for TensorFlow:
https://github.com/antmicro/litex-vexriscv-tensorflow-lite-demo
Building and Running
********************
The application can be built for the :ref:`litex-vexriscv` for
emulation in Renode as follows:
.. zephyr-app-commands::
:zephyr-app: samples/tensorflow/magic_wand
:host-os: unix
:board: litex_vexriscv
:goals: build
:compact:
Once the application is built, `download and install Renode 1.12 or higher as a package`_
following the instructions in the `Renode GitHub README`_ and
start the emulator:
.. code-block:: console
renode -e "set zephyr_elf @./build/zephyr/zephyr.elf; s @./renode/litex-vexriscv-tflite.resc"
.. _download and install Renode 1.12 or higher as a package:
https://github.com/renode/renode/releases/
.. _Renode GitHub README:
https://github.com/renode/renode/blob/master/README.rst
Sample Output
=============
The Renode-emulated LiteX/VexRiscv board is fed data that the
application recognizes as a series of alternating ring and slope
gestures.
.. code-block:: console
Got accelerometer, label: accel-0
RING:
*
* *
* *
* *
* *
* *
*
SLOPE:
*
*
*
*
*
*
*
* * * * * * * *
RING:
*
* *
* *
* *
* *
* *
*
SLOPE:
*
*
*
*
*
*
*
* * * * * * * *
Modifying Sample for Your Own Project
*************************************
It is recommended that you copy and modify one of the two TensorFlow
samples when creating your own TensorFlow project. To build with
TensorFlow, you must enable at least the below Kconfig options in
your :file:`prj.conf` and set a flag in your :file:`CMakeLists.txt`.
:file:`prj.conf`:
.. code-block:: console
CONFIG_CPLUSPLUS=y
CONFIG_NEWLIB_LIBC=y
CONFIG_TENSORFLOW_LITE_MICRO=y
:file:`CMakeLists.txt`:
.. code-block:: console
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-threadsafe-statics")
Training
********
Follow the instructions in the :file:`train/` directory to train your
own model for use in the sample.

17
samples/modules/tensorflow/magic_wand/boards/litex_vexriscv.conf Normal file
View file

@ -0,0 +1,17 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
CONFIG_ADXL345=y
CONFIG_PWM=n
CONFIG_PWM_LITEX=n

38
samples/modules/tensorflow/magic_wand/boards/litex_vexriscv.overlay Normal file
View file

@ -0,0 +1,38 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
&i2c0 {
label = "I2C0";
reg = <0xe0003000 0x4 0xe0003004 0x4>;
adxl@1d {
compatible = "adi,adxl345";
label = "accel-0";
reg = <0x1d>;
};
};
&pwm0 {
status = "disabled";
};
&eth0 {
status = "disabled";
};
&prbs0 {
status = "disabled";
};

22
samples/modules/tensorflow/magic_wand/prj.conf Normal file
View file

@ -0,0 +1,22 @@
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
CONFIG_CPLUSPLUS=y
CONFIG_LIB_CPLUSPLUS=y
CONFIG_NEWLIB_LIBC=y
CONFIG_NEWLIB_LIBC_FLOAT_PRINTF=y
CONFIG_SENSOR=y
CONFIG_NETWORKING=n
CONFIG_MAIN_STACK_SIZE=4096
CONFIG_TENSORFLOW_LITE_MICRO=y

128
samples/modules/tensorflow/magic_wand/renode/angle.data Normal file
View file

@ -0,0 +1,128 @@
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
-766 132 709
-751 249 659
-714 314 630
-709 244 623
-707 230 659
-704 202 748
-714 219 728
-722 239 710
-744 116 612
-753 -49 570
-748 -279 527
-668 -664 592
-601 -635 609
-509 -559 606
-286 -162 536
-255 -144 495
-209 -85 495
6 416 698
-33 304 1117
-82 405 1480
-198 1008 1908
-229 990 1743
-234 934 1453
-126 838 896
-78 792 911
-27 741 918
114 734 960
135 613 959
152 426 1015
106 -116 1110
63 -314 1129
-12 -486 1179
-118 -656 1510
-116 -558 1553
-126 -361 1367
-222 -76 922
-210 -26 971
-194 50 1053
-178 72 1082
-169 100 1073
-162 133 1050
-156 226 976
-154 323 886
-130 240 1154
-116 124 916
-132 124 937
-153 115 981
-184 94 962
-177 85 1017
-173 92 1027
-168 158 1110
-181 101 1030
-180 139 1054
-152 10 1044
-169 74 1007

128
samples/modules/tensorflow/magic_wand/renode/circle.data Normal file
View file

@ -0,0 +1,128 @@
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
0 0 0
-665 228 827
-680 339 716
-680 564 812
-679 552 818
-665 528 751
-658 432 618
-655 445 592
-667 484 556
-684 590 510
-674 672 475
-660 786 390
-562 1124 128
-526 1140 111
-486 1044 33
-416 652 -134
-390 534 -143
-365 381 -117
-314 60 94
-322 7 190
-338 -95 342
-360 -106 842
-351 -41 965
-352 12 960
-366 42 1124
-322 56 1178
-312 15 1338
-254 10 1532
-241 5 1590
-227 60 1565
-204 282 1560
-180 262 1524
-138 385 1522
-84 596 1626
-55 639 1604
-19 771 1511
16 932 1132
15 924 1013
1 849 812
-88 628 500
-114 609 463
-155 559 382
-234 420 278
-254 390 272
-327 200 336
-558 -556 630
-640 -607 740
-706 -430 868
-778 42 1042
-763 84 973
-735 185 931
-682 252 766
-673 230 757
-671 218 757
-656 222 714
-659 238 746
-640 276 731
-634 214 754
-637 207 735
-637 194 742
-634 248 716
-631 265 697
-628 252 797
-592 204 816
-618 218 812
-633 231 828
-640 222 736
-634 221 787

35
samples/modules/tensorflow/magic_wand/renode/litex-vexriscv-tflite.repl Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright (c) 2021 Antmicro
*
* SPDX-License-Identifier: Apache-2.0
*/
ram: Memory.MappedMemory @ {
sysbus 0x40000000;
sysbus 0xc0000000 // shadow
}
size: 0x10000000
cpu: CPU.VexRiscv @ sysbus
cpuType: "rv32imac"
uart: UART.LiteX_UART @ {
sysbus 0x60001800;
sysbus 0xE0001800 // shadow
}
-> cpu@2
timer0: Timers.LiteX_Timer @ {
sysbus 0x60002800;
sysbus 0xE0002800 // shadow
}
frequency: 100000000
-> cpu@1
i2c: I2C.LiteX_I2C_Zephyr @ {
sysbus 0x60003000;
sysbus 0xE0003000 // shadow
}
adxl345: Sensors.ADXL345 @ i2c 0x1D

33
samples/modules/tensorflow/magic_wand/renode/litex-vexriscv-tflite.resc Normal file
View file

@ -0,0 +1,33 @@
# Copyright (c) 2021 Antmicro
# SPDX-License-Identifier: Apache-2.0
using sysbus
$zephyr_elf?=@../build/zephyr/zephyr.elf
mach create
machine LoadPlatformDescription $ORIGIN/litex-vexriscv-tflite.repl
showAnalyzer uart
logLevel 3 i2c
macro reset
"""
sysbus LoadELF $zephyr_elf
i2c.adxl345 MaxFifoDepth 1
i2c.adxl345 FeedSample $ORIGIN/circle.data
i2c.adxl345 FeedSample 0 15000 15000 128
i2c.adxl345 FeedSample 0 0 0 128
i2c.adxl345 FeedSample $ORIGIN/angle.data
i2c.adxl345 FeedSample 0 15000 15000 128
i2c.adxl345 FeedSample 0 0 0 128
i2c.adxl345 FeedSample $ORIGIN/circle.data
i2c.adxl345 FeedSample 0 15000 15000 128
i2c.adxl345 FeedSample 0 0 0 128
i2c.adxl345 FeedSample $ORIGIN/angle.data
i2c.adxl345 FeedSample 0 15000 15000 128
i2c.adxl345 FeedSample 0 0 0 128
"""
runMacro $reset

107
samples/modules/tensorflow/magic_wand/renode/litex-vexriscv-tflite.robot Normal file
View file

@ -0,0 +1,107 @@
# Copyright (c) 2021 Antmicro
# SPDX-License-Identifier: Apache-2.0
*** Settings ***
Suite Setup Run Keywords
... Setup AND
... Execute Command include @${CURDIR}/LiteX_I2C_Zephyr.cs
Suite Teardown Teardown
Test Setup Reset Emulation
Resource ${RENODEKEYWORDS}
*** Keywords ***
Wait For Ring
Wait For Line On Uart RING:
# Passing whitespaces in arguments is a bit tricky.
# Here we wait for the following pattern:
# *
# * *
# * *
# * *
# * *
# * *
# *
Wait For Line On Uart ${SPACE*10}*
Wait For Line On Uart ${SPACE*7}*${SPACE*5}*
Wait For Line On Uart ${SPACE*5}*${SPACE*9}*
Wait For Line On Uart ${SPACE*4}*${SPACE*11}*
Wait For Line On Uart ${SPACE*5}*${SPACE*9}*
Wait For Line On Uart ${SPACE*7}*${SPACE*5}*
Wait For Line On Uart ${SPACE*10}*
Wait For Slope
Wait For Line On Uart SLOPE:
# Passing whitespaces in arguments is a bit tricky.
# Here we wait for the following pattern:
# *
# *
# *
# *
# *
# *
# *
# * * * * * * * *
Wait For Line On Uart ${SPACE*8}*
Wait For Line On Uart ${SPACE*7}*
Wait For Line On Uart ${SPACE*6}*
Wait For Line On Uart ${SPACE*5}*
Wait For Line On Uart ${SPACE*4}*
Wait For Line On Uart ${SPACE*3}*
Wait For Line On Uart ${SPACE*2}*
Wait For Line On Uart ${SPACE}* * * * * * * *
*** Test Cases ***
Run Hello World Demo
Execute Command using sysbus
Execute Command mach create
Execute Command machine LoadPlatformDescription @${CURDIR}/litex-vexriscv-tflite.repl
Execute Command showAnalyzer uart Antmicro.Renode.Analyzers.LoggingUartAnalyzer
Execute Command sysbus LoadELF @${CURDIR}/../tensorflow/tensorflow/lite/micro/tools/make/gen/zephyr_vexriscv_x86_64/hello_world/build/zephyr/zephyr.elf
Create Terminal Tester sysbus.uart
Start Emulation
Wait For Line On Uart Booting Zephyr OS
Wait For Line On Uart x_value
Wait For Line On Uart y_value
Run Magic Wand Demo
Execute Command using sysbus
Execute Command mach create
Execute Command machine LoadPlatformDescription @${CURDIR}/litex-vexriscv-tflite.repl
Execute Command showAnalyzer uart Antmicro.Renode.Analyzers.LoggingUartAnalyzer
Execute Command sysbus LoadELF @${CURDIR}/../tensorflow/tensorflow/lite/micro/tools/make/gen/zephyr_vexriscv_x86_64/magic_wand/build/zephyr/zephyr.elf
Execute Command i2c.adxl345 MaxFifoDepth 1
Execute Command i2c.adxl345 FeedSample @${CURDIR}/circle.data
Execute Command i2c.adxl345 FeedSample 0 15000 15000 128
Execute Command i2c.adxl345 FeedSample 0 0 0 128
Execute Command i2c.adxl345 FeedSample @${CURDIR}/angle.data
Execute Command i2c.adxl345 FeedSample 0 15000 15000 128
Execute Command i2c.adxl345 FeedSample 0 0 0 128
Execute Command i2c.adxl345 FeedSample @${CURDIR}/circle.data
Execute Command i2c.adxl345 FeedSample 0 15000 15000 128
Execute Command i2c.adxl345 FeedSample 0 0 0 128
Execute Command i2c.adxl345 FeedSample @${CURDIR}/angle.data
Execute Command i2c.adxl345 FeedSample 0 15000 15000 128
Execute Command i2c.adxl345 FeedSample 0 0 0 128
Create Terminal Tester sysbus.uart timeout=480
Start Emulation
Wait For Line On Uart Booting Zephyr OS
Wait For Line On Uart Got accelerometer
Wait For Ring
Wait For Slope
Wait For Ring
Wait For Slope

8
samples/modules/tensorflow/magic_wand/sample.yaml Normal file
View file

@ -0,0 +1,8 @@
sample:
description: Magic Wand TensorFlow sample
name: magic wand tensorflow
tests:
sample.tensorflow.magicwand:
platform_allow: litex_vexriscv
build_only: true
tags: tensorflow

105
samples/modules/tensorflow/magic_wand/src/accelerometer_handler.cc Normal file
View file

@ -0,0 +1,105 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "accelerometer_handler.h"
#include <device.h>
#include <drivers/sensor.h>
#include <stdio.h>
#include <string.h>
#include <zephyr.h>
#define BUFLEN 300
int begin_index = 0;
const char *label = NULL;
const struct device *sensor = NULL;
int current_index = 0;
float bufx[BUFLEN] = { 0.0f };
float bufy[BUFLEN] = { 0.0f };
float bufz[BUFLEN] = { 0.0f };
bool initial = true;
TfLiteStatus SetupAccelerometer(tflite::ErrorReporter *error_reporter)
{
label = DT_LABEL(DT_INST(0, adi_adxl345));
sensor = device_get_binding(label);
if (sensor == NULL) {
TF_LITE_REPORT_ERROR(error_reporter,
"Failed to get accelerometer, label: %s\n",
label);
} else {
TF_LITE_REPORT_ERROR(error_reporter, "Got accelerometer, label: %s\n",
label);
}
return kTfLiteOk;
}
bool ReadAccelerometer(tflite::ErrorReporter *error_reporter, float *input,
int length)
{
int rc;
struct sensor_value accel[3];
int samples_count;
rc = sensor_sample_fetch(sensor);
if (rc < 0) {
TF_LITE_REPORT_ERROR(error_reporter, "Fetch failed\n");
return false;
}
/* Skip if there is no data */
if (!rc) {
return false;
}
samples_count = rc;
for (int i = 0; i < samples_count; i++) {
rc = sensor_channel_get(sensor, SENSOR_CHAN_ACCEL_XYZ, accel);
if (rc < 0) {
TF_LITE_REPORT_ERROR(error_reporter, "ERROR: Update failed: %d\n", rc);
return false;
}
bufx[begin_index] = (float)sensor_value_to_double(&accel[0]);
bufy[begin_index] = (float)sensor_value_to_double(&accel[1]);
bufz[begin_index] = (float)sensor_value_to_double(&accel[2]);
begin_index++;
if (begin_index >= BUFLEN) {
begin_index = 0;
}
}
if (initial && begin_index >= 100) {
initial = false;
}
if (initial) {
return false;
}
int sample = 0;
for (int i = 0; i < (length - 3); i += 3) {
int ring_index = begin_index + sample - length / 3;
if (ring_index < 0) {
ring_index += BUFLEN;
}
input[i] = bufx[ring_index];
input[i + 1] = bufy[ring_index];
input[i + 2] = bufz[ring_index];
sample++;
}
return true;
}

30
samples/modules/tensorflow/magic_wand/src/accelerometer_handler.h Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_ACCELEROMETER_HANDLER_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_ACCELEROMETER_HANDLER_H_
#define kChannelNumber 3
#include <tensorflow/lite/c/c_api_internal.h>
#include <tensorflow/lite/micro/micro_error_reporter.h>
extern int begin_index;
extern TfLiteStatus SetupAccelerometer(tflite::ErrorReporter *error_reporter);
extern bool ReadAccelerometer(tflite::ErrorReporter *error_reporter,
float *input, int length);
#endif /* TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_ACCELEROMETER_HANDLER_H_ */

23
samples/modules/tensorflow/magic_wand/src/assert.cc Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
extern "C" {
void __assert_func(const char *, int, const char *, const char *)
{
}
}

40
samples/modules/tensorflow/magic_wand/src/constants.h Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_CONSTANTS_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_CONSTANTS_H_
/* The expected accelerometer data sample frequency */
const float kTargetHz = 25;
/* What gestures are supported. */
constexpr int kGestureCount = 4;
constexpr int kWingGesture = 0;
constexpr int kRingGesture = 1;
constexpr int kSlopeGesture = 2;
constexpr int kNoGesture = 3;
/* These control the sensitivity of the detection algorithm. If you're seeing
* too many false positives or not enough true positives, you can try tweaking
* these thresholds. Often, increasing the size of the training set will give
* more robust results though, so consider retraining if you are seeing poor
* predictions.
*/
constexpr float kDetectionThreshold = 0.8f;
constexpr int kPredictionHistoryLength = 5;
constexpr int kPredictionSuppressionDuration = 25;
#endif /* TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_CONSTANTS_H_ */

78
samples/modules/tensorflow/magic_wand/src/gesture_predictor.cc Normal file
View file

@ -0,0 +1,78 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "gesture_predictor.h"
#include "constants.h"
namespace {
/* State for the averaging algorithm we're using. */
float prediction_history[kGestureCount][kPredictionHistoryLength] = {};
int prediction_history_index = 0;
int prediction_suppression_count = 0;
} /* namespace */
/* Return the result of the last prediction
* 0: wing("W"), 1: ring("O"), 2: slope("angle"), 3: unknown
*/
int PredictGesture(float *output)
{
/* Record the latest predictions in our rolling history buffer. */
for (int i = 0; i < kGestureCount; ++i) {
prediction_history[i][prediction_history_index] = output[i];
}
/* Figure out which slot to put the next predictions into. */
++prediction_history_index;
if (prediction_history_index >= kPredictionHistoryLength) {
prediction_history_index = 0;
}
/* Average the last n predictions for each gesture, and find which has the
* highest score.
*/
int max_predict_index = -1;
float max_predict_score = 0.0f;
for (int i = 0; i < kGestureCount; i++) {
float prediction_sum = 0.0f;
for (int j = 0; j < kPredictionHistoryLength; ++j) {
prediction_sum += prediction_history[i][j];
}
const float prediction_average = prediction_sum / kPredictionHistoryLength;
if ((max_predict_index == -1) || (prediction_average > max_predict_score)) {
max_predict_index = i;
max_predict_score = prediction_average;
}
}
/* If there's been a recent prediction, don't trigger a new one too soon. */
if (prediction_suppression_count > 0) {
--prediction_suppression_count;
}
/* If we're predicting no gesture, or the average score is too low, or there's
* been a gesture recognised too recently, return no gesture.
*/
if ((max_predict_index == kNoGesture) ||
(max_predict_score < kDetectionThreshold) ||
(prediction_suppression_count > 0)) {
return kNoGesture;
} else {
/* Reset the suppression counter so we don't come up with another prediction
* too soon.
*/
prediction_suppression_count = kPredictionSuppressionDuration;
return max_predict_index;
}
}

22
samples/modules/tensorflow/magic_wand/src/gesture_predictor.h Normal file
View file

@ -0,0 +1,22 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_GESTURE_PREDICTOR_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_GESTURE_PREDICTOR_H_
extern int PredictGesture(float *output);
#endif /* TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_GESTURE_PREDICTOR_H_ */

1662
samples/modules/tensorflow/magic_wand/src/magic_wand_model_data.cc Normal file
View file

File diff suppressed because it is too large Load diff

29
samples/modules/tensorflow/magic_wand/src/magic_wand_model_data.h Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* This is a standard TensorFlow Lite model file that has been converted into a
* C data array, so it can be easily compiled into a binary for devices that
* don't have a file system. It was created using the command:
* xxd -i magic_wand_model.tflite > magic_wand_model_data.cc
*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_MAGIC_WAND_MODEL_DATA_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_MAGIC_WAND_MODEL_DATA_H_
extern const unsigned char g_magic_wand_model_data[];
extern const int g_magic_wand_model_data_len;
#endif /* TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_MAGIC_WAND_MODEL_DATA_H_ */

30
samples/modules/tensorflow/magic_wand/src/main.cc Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "main_functions.h"
/* This is the default main used on systems that have the standard C entry
* point. Other devices (for example FreeRTOS or ESP32) that have different
* requirements for entry code (like an app_main function) should specialize
* this main.cc file in a target-specific subfolder.
*/
int main(int argc, char *argv[])
{
setup();
while (true) {
loop();
}
}

131
samples/modules/tensorflow/magic_wand/src/main_functions.cc Normal file
View file

@ -0,0 +1,131 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "main_functions.h"
#include "accelerometer_handler.h"
#include "constants.h"
#include "gesture_predictor.h"
#include "magic_wand_model_data.h"
#include "output_handler.h"
#include <tensorflow/lite/micro/micro_error_reporter.h>
#include <tensorflow/lite/micro/micro_interpreter.h>
#include <tensorflow/lite/micro/micro_mutable_op_resolver.h>
#include <tensorflow/lite/schema/schema_generated.h>
#include <tensorflow/lite/version.h>
/* Globals, used for compatibility with Arduino-style sketches. */
namespace {
tflite::ErrorReporter *error_reporter = nullptr;
const tflite::Model *model = nullptr;
tflite::MicroInterpreter *interpreter = nullptr;
TfLiteTensor *model_input = nullptr;
int input_length;
/* Create an area of memory to use for input, output, and intermediate arrays.
* The size of this will depend on the model you're using, and may need to be
* determined by experimentation.
*/
constexpr int kTensorArenaSize = 60 * 1024;
uint8_t tensor_arena[kTensorArenaSize];
} /* namespace */
/* The name of this function is important for Arduino compatibility. */
void setup(void)
{
/* Set up logging. Google style is to avoid globals or statics because of
* lifetime uncertainty, but since this has a trivial destructor it's okay.
*/
static tflite::MicroErrorReporter micro_error_reporter; /* NOLINT */
error_reporter = &micro_error_reporter;
/* Map the model into a usable data structure. This doesn't involve any
* copying or parsing, it's a very lightweight operation.
*/
model = tflite::GetModel(g_magic_wand_model_data);
if (model->version() != TFLITE_SCHEMA_VERSION) {
TF_LITE_REPORT_ERROR(error_reporter,
"Model provided is schema version %d not equal "
"to supported version %d.",
model->version(), TFLITE_SCHEMA_VERSION);
return;
}
/* Pull in only the operation implementations we need.
* This relies on a complete list of all the ops needed by this graph.
* An easier approach is to just use the AllOpsResolver, but this will
* incur some penalty in code space for op implementations that are not
* needed by this graph.
*/
static tflite::MicroMutableOpResolver < 5 > micro_op_resolver; /* NOLINT */
micro_op_resolver.AddConv2D();
micro_op_resolver.AddDepthwiseConv2D();
micro_op_resolver.AddFullyConnected();
micro_op_resolver.AddMaxPool2D();
micro_op_resolver.AddSoftmax();
/* Build an interpreter to run the model with. */
static tflite::MicroInterpreter static_interpreter(
model, micro_op_resolver, tensor_arena, kTensorArenaSize, error_reporter);
interpreter = &static_interpreter;
/* Allocate memory from the tensor_arena for the model's tensors. */
interpreter->AllocateTensors();
/* Obtain pointer to the model's input tensor. */
model_input = interpreter->input(0);
if ((model_input->dims->size != 4) || (model_input->dims->data[0] != 1) ||
(model_input->dims->data[1] != 128) ||
(model_input->dims->data[2] != kChannelNumber) ||
(model_input->type != kTfLiteFloat32)) {
TF_LITE_REPORT_ERROR(error_reporter,
"Bad input tensor parameters in model");
return;
}
input_length = model_input->bytes / sizeof(float);
TfLiteStatus setup_status = SetupAccelerometer(error_reporter);
if (setup_status != kTfLiteOk) {
TF_LITE_REPORT_ERROR(error_reporter, "Set up failed\n");
}
}
void loop(void)
{
/* Attempt to read new data from the accelerometer. */
bool got_data =
ReadAccelerometer(error_reporter, model_input->data.f, input_length);
/* If there was no new data, wait until next time. */
if (!got_data) {
return;
}
/* Run inference, and report any error */
TfLiteStatus invoke_status = interpreter->Invoke();
if (invoke_status != kTfLiteOk) {
TF_LITE_REPORT_ERROR(error_reporter, "Invoke failed on index: %d\n",
begin_index);
return;
}
/* Analyze the results to obtain a prediction */
int gesture_index = PredictGesture(interpreter->output(0)->data.f);
/* Produce an output */
HandleOutput(error_reporter, gesture_index);
}

40
samples/modules/tensorflow/magic_wand/src/main_functions.h Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_MAIN_FUNCTIONS_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_MAIN_FUNCTIONS_H_
/* Expose a C friendly interface for main functions.*/
#ifdef __cplusplus
extern "C" {
#endif
/* Initializes all data needed for the example. The name is important, and needs
* to be setup() for Arduino compatibility.
*/
void setup(void);
/* Runs one iteration of data gathering and inference. This should be called
* repeatedly from the application code. The name needs to be loop() for Arduino
* compatibility.
*/
void loop(void);
#ifdef __cplusplus
}
#endif
#endif /* TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_MAIN_FUNCTIONS_H_ */

40
samples/modules/tensorflow/magic_wand/src/output_handler.cc Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "output_handler.h"
void HandleOutput(tflite::ErrorReporter *error_reporter, int kind)
{
/* light (red: wing, blue: ring, green: slope) */
if (kind == 0) {
TF_LITE_REPORT_ERROR(
error_reporter,
"WING:\n\r* * *\n\r * * * "
"*\n\r * * * *\n\r * * * *\n\r * * "
"* *\n\r * *\n\r");
} else if (kind == 1) {
TF_LITE_REPORT_ERROR(
error_reporter,
"RING:\n\r *\n\r * *\n\r * *\n\r "
" * *\n\r * *\n\r * *\n\r "
" *\n\r");
} else if (kind == 2) {
TF_LITE_REPORT_ERROR(
error_reporter,
"SLOPE:\n\r *\n\r *\n\r *\n\r *\n\r "
"*\n\r *\n\r *\n\r * * * * * * * *\n\r");
}
}

25
samples/modules/tensorflow/magic_wand/src/output_handler.h Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_OUTPUT_HANDLER_H_
#define TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_OUTPUT_HANDLER_H_
#include <tensorflow/lite/c/common.h>
#include <tensorflow/lite/micro/micro_error_reporter.h>
void HandleOutput(tflite::ErrorReporter *error_reporter, int kind);
#endif /* TENSORFLOW_LITE_MICRO_EXAMPLES_MAGIC_WAND_OUTPUT_HANDLER_H_ */