tests: bsim_bt: Add EDTT based BT stack tests

Add some scripts to automatically run a set of BT conformance tests. Each script has an associated file which selects which subset of tests are run by that script. The LL scripts are divided in 2 subsets so as to allow parallelizing a bit the run (the LL tests take the longest). Except these, all other sets are just divided by category. Signed-off-by: Alberto Escolar Piedras <alpi@oticon.com>
2019-06-25 15:48:00 +02:00 · 2019-06-25 15:48:00 +02:00 · 0a21d3c455
commit 0a21d3c455
parent 8be3f983ea
11 changed files with 440 additions and 0 deletions
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/_controller_tests_inner.sh
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/_controller_tests_inner.sh
@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+# Common part of the test scripts for some of the EDTT tests
+# in which 2 controller only builds of the stack are run against each other
+VERBOSITY_LEVEL=2
+PROCESS_IDS=""; EXIT_CODE=0
+
+function Execute(){
+  if [ ! -f $1 ]; then
+    echo -e "  \e[91m`pwd`/`basename $1` cannot be found (did you forget to\
+ compile it?)\e[39m"
+    exit 1
+  fi
+  timeout 300 $@ & PROCESS_IDS="$PROCESS_IDS $!"
+}
+
+: "${BSIM_OUT_PATH:?BSIM_OUT_PATH must be defined}"
+: "${EDTT_PATH:?EDTT_PATH must be defined}"
+
+#Give a default value to BOARD if it does not have one yet:
+BOARD="${BOARD:-nrf52_bsim}"
+
+cd ${EDTT_PATH}
+
+Execute ./src/edttool.py -s=${SIMULATION_ID} -d=0 --transport bsim \
+  -T $TEST_MODULE -C $TEST_FILE -v=${VERBOSITY_LEVEL}
+
+cd ${BSIM_OUT_PATH}/bin
+
+Execute ./bs_device_EDTT_bridge -s=${SIMULATION_ID} -d=0 -AutoTerminate \
+  -RxWait=2.5e3 -D=2 -dev0=1 -dev1=2 -v=${VERBOSITY_LEVEL}
+
+Execute \
+  ./bs_${BOARD}_tests_bluetooth_bsim_bt_edtt_ble_test_app_hci_test_app_prj_conf\
+  -s=${SIMULATION_ID} -d=1 -v=${VERBOSITY_LEVEL} -RealEncryption=1
+
+Execute \
+  ./bs_${BOARD}_tests_bluetooth_bsim_bt_edtt_ble_test_app_hci_test_app_prj_conf\
+  -s=${SIMULATION_ID} -d=2 -v=${VERBOSITY_LEVEL} -RealEncryption=1
+
+Execute ./bs_2G4_phy_v1 -v=${VERBOSITY_LEVEL} -s=${SIMULATION_ID} \
+  -D=3 -sim_length=3600e6 $@
+
+for PROCESS_ID in $PROCESS_IDS; do
+  wait $PROCESS_ID || let "EXIT_CODE=$?"
+done
+exit $EXIT_CODE #the last exit code != 0
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gap.sh
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gap.sh
@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+# GAP regression tests based on the EDTTool
+CWD=`pwd`"/"`dirname $0`
+
+export SIMULATION_ID="edtt_gap"
+export TEST_FILE=${CWD}"/gap.test_list"
+export TEST_MODULE="gap_verification"
+
+${CWD}/_controller_tests_inner.sh
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gap.test_list
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gap.test_list
@ -0,0 +1,58 @@
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+GAP/ADV/BV-01-C
+GAP/ADV/BV-02-C
+GAP/ADV/BV-03-C
+GAP/ADV/BV-04-C
+GAP/ADV/BV-05-C
+GAP/ADV/BV-08-C
+GAP/ADV/BV-09-C
+GAP/ADV/BV-10-C
+GAP/ADV/BV-11-C
+GAP/ADV/BV-12-C
+GAP/ADV/BV-13-C
+GAP/ADV/BV-14-C
+GAP/ADV/BV-15-C
+GAP/ADV/BV-16-C
+GAP/ADV/BV-17-C
+GAP/CONN/ACEP/BV-01-C
+GAP/CONN/ACEP/BV-03-C
+GAP/CONN/ACEP/BV-04-C
+GAP/CONN/DCON/BV-01-C
+GAP/CONN/ENC
+GAP/CONN/GCEP/BV-01-C
+GAP/CONN/GCEP/BV-02-C
+GAP/CONN/GCEP/BV-05-C
+GAP/CONN/GCEP/BV-06-C
+GAP/CONN/CPUP/BV-01-C
+GAP/CONN/CPUP/BV-02-C
+GAP/CONN/CPUP/BV-03-C
+GAP/CONN/CPUP/BV-04-C
+GAP/CONN/CPUP/BV-05-C
+GAP/CONN/CPUP/BV-06-C
+GAP/CONN/NCON/BV-01-C
+GAP/CONN/NCON/BV-02-C
+GAP/CONN/NCON/BV-03-C
+GAP/CONN/UCON/BV-01-C
+GAP/CONN/UCON/BV-02-C
+GAP/CONN/UCON/BV-03-C
+GAP/DISC/GENM/BV-01-C
+GAP/DISC/GENM/BV-02-C
+GAP/DISC/GENM/BV-03-C
+GAP/DISC/GENM/BV-04-C
+GAP/DISC/GENP/BV-01-C
+GAP/DISC/GENP/BV-02-C
+GAP/DISC/GENP/BV-04-C
+GAP/DISC/GENP/BV-05-C
+GAP/DISC/LIMM/BV-01-C
+GAP/DISC/LIMM/BV-02-C
+GAP/DISC/LIMM/BV-03-C
+GAP/DISC/LIMM/BV-04-C
+GAP/DISC/LIMP/BV-01-C
+GAP/DISC/LIMP/BV-02-C
+GAP/DISC/LIMP/BV-04-C
+GAP/DISC/LIMP/BV-05-C
+GAP/DISC/NONM/BV-01-C
+GAP/DISC/NONM/BV-02-C
+GAP/DISC/RPA/BV-01-C
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gatt.sh
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gatt.sh
@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+# GATT regression tests based on the EDTTool
+SIMULATION_ID="edtt_gatt"
+VERBOSITY_LEVEL=2
+PROCESS_IDS=""; EXIT_CODE=0
+CWD=`pwd`"/"`dirname $0`
+
+function Execute(){
+  if [ ! -f $1 ]; then
+    echo -e "  \e[91m`pwd`/`basename $1` cannot be found (did you forget to\
+ compile it?)\e[39m"
+    exit 1
+  fi
+  timeout 300 $@ & PROCESS_IDS="$PROCESS_IDS $!"
+}
+
+: "${BSIM_OUT_PATH:?BSIM_OUT_PATH must be defined}"
+: "${EDTT_PATH:?EDTT_PATH must be defined}"
+
+#Give a default value to BOARD if it does not have one yet:
+BOARD="${BOARD:-nrf52_bsim}"
+
+cd ${EDTT_PATH}
+
+Execute ./src/edttool.py -s=${SIMULATION_ID} -d=0 --transport bsim \
+  -T gatt_verification -C "${CWD}/gatt.test_list" -v=${VERBOSITY_LEVEL}
+
+cd ${BSIM_OUT_PATH}/bin
+
+Execute ./bs_device_EDTT_bridge -s=${SIMULATION_ID} -d=0 -AutoTerminate \
+  -RxWait=2.5e3 -D=2 -dev0=1 -dev1=2 -v=${VERBOSITY_LEVEL}
+
+Execute \
+ ./bs_${BOARD}_tests_bluetooth_bsim_bt_edtt_ble_test_app_hci_test_app_prj_conf\
+  -s=${SIMULATION_ID} -d=1 -v=${VERBOSITY_LEVEL} -RealEncryption=1
+
+Execute \
+ ./bs_${BOARD}_tests_bluetooth_bsim_bt_edtt_ble_test_app_gatt_test_app_prj_conf\
+  -s=${SIMULATION_ID} -d=2 -v=${VERBOSITY_LEVEL} -RealEncryption=1
+
+Execute ./bs_2G4_phy_v1 -v=${VERBOSITY_LEVEL} -s=${SIMULATION_ID} \
+  -D=3 -sim_length=3600e6 $@
+
+for PROCESS_ID in $PROCESS_IDS; do
+  wait $PROCESS_ID || let "EXIT_CODE=$?"
+done
+exit $EXIT_CODE #the last exit code != 0
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gatt.test_list
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/gatt.test_list
@ -0,0 +1,76 @@
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+GAP/GAT/BV-01-C
+GAP/GAT/BV-02-C
+GAP/GAT/BV-03-C
+GAP/GAT/BV-04-C
+GAP/GAT/BV-05-C
+GAP/IDLE/NAMP/BV-01-C
+GAP/GAT/BX-01-C
+GATT/SR/GAC/BV-01-C
+GATT/SR/GAD/BV-01-C
+GATT/SR/GAD/BV-02-C
+GATT/SR/GAD/BV-03-C
+GATT/SR/GAD/BV-04-C
+GATT/SR/GAD/BV-05-C
+GATT/SR/GAD/BV-06-C
+GATT/SR/GAR/BV-01-C
+GATT/SR/GAR/BI-01-C
+GATT/SR/GAR/BI-02-C
+GATT/SR/GAR/BI-03-C
+GATT/SR/GAR/BI-04-C
+GATT/SR/GAR/BV-03-C
+GATT/SR/GAR/BI-06-C
+GATT/SR/GAR/BI-07-C
+GATT/SR/GAR/BI-09-C
+GATT/SR/GAR/BI-10-C
+GATT/SR/GAR/BI-11-C
+GATT/SR/GAR/BV-04-C
+GATT/SR/GAR/BI-12-C
+GATT/SR/GAR/BI-13-C
+GATT/SR/GAR/BI-14-C
+GATT/SR/GAR/BI-15-C
+GATT/SR/GAR/BI-16-C
+GATT/SR/GAR/BI-17-C
+GATT/SR/GAR/BV-05-C
+GATT/SR/GAR/BI-18-C
+GATT/SR/GAR/BI-19-C
+GATT/SR/GAR/BI-20-C
+GATT/SR/GAR/BI-21-C
+GATT/SR/GAR/BI-22-C
+GATT/SR/GAR/BV-06-C
+GATT/SR/GAR/BV-07-C
+GATT/SR/GAW/BV-01-C
+GATT/SR/GAW/BV-03-C
+GATT/SR/GAW/BI-02-C
+GATT/SR/GAW/BI-03-C
+GATT/SR/GAW/BI-04-C
+GATT/SR/GAW/BI-05-C
+GATT/SR/GAW/BI-06-C
+GATT/SR/GAW/BV-05-C
+GATT/SR/GAW/BI-07-C
+GATT/SR/GAW/BI-08-C
+GATT/SR/GAW/BI-09-C
+GATT/SR/GAW/BI-11-C
+GATT/SR/GAW/BI-12-C
+GATT/SR/GAW/BI-13-C
+GATT/SR/GAW/BV-08-C
+GATT/SR/GAW/BV-09-C
+GATT/SR/GAW/BV-11-C
+GATT/SR/GAW/BI-32-C
+GATT/SR/GAW/BI-33-C
+GATT/SR/GAN/BV-01-C
+GATT/SR/GAI/BV-01-C
+GATT/SR/GAS/BV-01-C
+GATT/SR/UNS/BI-01-C
+GATT/SR/UNS/BI-02-C
+GATT/SR/GPA/BV-01-C
+GATT/SR/GPA/BV-02-C
+GATT/SR/GPA/BV-03-C
+GATT/SR/GPA/BV-04-C
+GATT/SR/GPA/BV-05-C
+GATT/SR/GPA/BV-06-C
+GATT/SR/GPA/BV-07-C
+GATT/SR/GPA/BV-08-C
+GATT/SR/GPA/BV-12-C
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/hci.sh
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/hci.sh
@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+# HCI regression tests based on the EDTTool
+CWD=`pwd`"/"`dirname $0`
+
+export SIMULATION_ID="edtt_hci"
+export TEST_FILE=${CWD}"/hci.test_list"
+export TEST_MODULE="hci_verification"
+
+${CWD}/_controller_tests_inner.sh
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/hci.test_list
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/hci.test_list
@ -0,0 +1,31 @@
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+HCI/CCO/BV-07-C
+HCI/CCO/BV-09-C
+HCI/CCO/BV-10-C
+HCI/CCO/BV-11-C
+HCI/CCO/BV-12-C
+HCI/CCO/BV-13-C
+HCI/CCO/BV-14-C
+HCI/CCO/BV-15-C
+HCI/CCO/BV-18-C
+HCI/CFC/BV-02-C
+HCI/CIN/BV-01-C
+HCI/CIN/BV-03-C
+HCI/CIN/BV-04-C
+HCI/CIN/BV-06-C
+HCI/CIN/BV-09-C
+HCI/CM/BV-01-C
+HCI/CM/BV-02-C
+HCI/CM/BV-03-C
+HCI/DDI/BI-02-C
+HCI/DDI/BV-03-C
+HCI/DDI/BV-04-C
+HCI/DSU/BV-02-C
+HCI/DSU/BV-03-C
+HCI/DSU/BV-04-C
+HCI/DSU/BV-05-C
+HCI/DSU/BV-06-C
+HCI/GEV/BV-01-C
+HCI/HFC/BV-04-C
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.1.sh
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.1.sh
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+# LL regression tests based on the EDTTool (part 1)
+
+CWD=`pwd`"/"`dirname $0`
+
+export SIMULATION_ID="edtt_ll_set1"
+export TEST_FILE=${CWD}"/ll.set1.test_list"
+export TEST_MODULE="ll_verification"
+
+${CWD}/_controller_tests_inner.sh
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.2.sh
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.2.sh
@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+# LL regression tests based on the EDTTool (part 2)
+
+CWD=`pwd`"/"`dirname $0`
+
+export SIMULATION_ID="edtt_ll_set2"
+export TEST_FILE=${CWD}"/ll.set2.test_list"
+export TEST_MODULE="ll_verification"
+
+${CWD}/_controller_tests_inner.sh
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.set1.test_list
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.set1.test_list
@ -0,0 +1,64 @@
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+LL/CON/ADV/BV-01-C
+LL/CON/ADV/BV-04-C
+LL/CON/ADV/BV-09-C
+LL/CON/ADV/BV-10-C
+LL/CON/INI/BV-01-C
+LL/CON/INI/BV-02-C
+LL/CON/INI/BV-06-C
+LL/CON/INI/BV-07-C
+LL/CON/INI/BV-08-C
+LL/CON/INI/BV-09-C
+LL/CON/INI/BV-10-C
+LL/CON/INI/BV-11-C
+LL/CON/INI/BV-12-C
+LL/CON/INI/BV-16-C
+LL/CON/INI/BV-17-C
+LL/CON/INI/BV-18-C
+LL/CON/INI/BV-19-C
+LL/CON/INI/BV-20-C
+LL/CON/INI/BV-21-C
+LL/CON/INI/BV-23-C
+#LL/CON/INI/BV-24-C #currently failing, to be investigated
+LL/CON/MAS/BI-06-C
+LL/CON/MAS/BV-03-C
+LL/CON/MAS/BV-04-C
+LL/CON/MAS/BV-05-C
+LL/CON/MAS/BV-07-C
+LL/CON/MAS/BV-08-C
+LL/CON/MAS/BV-09-C
+LL/CON/MAS/BV-13-C
+LL/CON/MAS/BV-20-C
+LL/CON/MAS/BV-21-C
+LL/CON/MAS/BV-23-C
+LL/CON/MAS/BV-24-C
+LL/CON/MAS/BV-25-C
+LL/CON/MAS/BV-26-C
+LL/CON/MAS/BV-27-C
+LL/CON/MAS/BV-29-C
+LL/CON/MAS/BV-30-C
+LL/CON/MAS/BV-34-C
+LL/CON/MAS/BV-35-C
+LL/CON/MAS/BV-41-C
+LL/CON/MAS/BV-43-C
+LL/CON/MAS/BV-73-C
+LL/CON/MAS/BV-74-C
+LL/CON/MAS/BV-76-C
+LL/CON/MAS/BV-77-C
+LL/CON/SLA/BI-08-C
+LL/CON/SLA/BV-04-C
+LL/CON/SLA/BV-05-C
+LL/CON/SLA/BV-06-C
+LL/CON/SLA/BV-10-C
+LL/CON/SLA/BV-11-C
+LL/CON/SLA/BV-12-C
+LL/CON/SLA/BV-14-C
+LL/CON/SLA/BV-19-C
+LL/CON/SLA/BV-20-C
+LL/CON/SLA/BV-22-C
+LL/CON/SLA/BV-24-C
+LL/CON/SLA/BV-25-C
+LL/CON/SLA/BV-26-C
+LL/CON/SLA/BV-27-C
--- a/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.set2.test_list
+++ b/tests/bluetooth/bsim_bt/edtt_ble_test_app/tests_scripts/ll.set2.test_list
@ -0,0 +1,62 @@
+# Copyright 2019 Oticon A/S
+# SPDX-License-Identifier: Apache-2.0
+
+LL/CON/SLA/BV-29-C
+LL/CON/SLA/BV-33-C
+LL/CON/SLA/BV-34-C
+LL/CON/SLA/BV-40-C
+LL/CON/SLA/BV-42-C
+LL/CON/SLA/BV-77-C
+LL/CON/SLA/BV-78-C
+LL/CON/SLA/BV-80-C
+LL/CON/SLA/BV-81-C
+LL/DDI/ADV/BV-01-C
+LL/DDI/ADV/BV-02-C
+LL/DDI/ADV/BV-03-C
+LL/DDI/ADV/BV-04-C
+LL/DDI/ADV/BV-05-C
+LL/DDI/ADV/BV-06-C
+LL/DDI/ADV/BV-07-C
+LL/DDI/ADV/BV-08-C
+LL/DDI/ADV/BV-09-C
+LL/DDI/ADV/BV-11-C
+LL/DDI/ADV/BV-15-C
+LL/DDI/ADV/BV-16-C
+LL/DDI/ADV/BV-17-C
+LL/DDI/ADV/BV-18-C
+LL/DDI/ADV/BV-19-C
+LL/DDI/ADV/BV-20-C
+LL/DDI/SCN/BV-01-C
+LL/DDI/SCN/BV-02-C
+LL/DDI/SCN/BV-03-C
+LL/DDI/SCN/BV-04-C
+LL/DDI/SCN/BV-05-C
+LL/DDI/SCN/BV-10-C
+LL/DDI/SCN/BV-11-C
+LL/DDI/SCN/BV-12-C
+LL/DDI/SCN/BV-13-C
+LL/DDI/SCN/BV-14-C
+LL/DDI/SCN/BV-15-C
+LL/DDI/SCN/BV-16-C
+LL/DDI/SCN/BV-17-C
+LL/DDI/SCN/BV-18-C
+LL/DDI/SCN/BV-26-C
+LL/DDI/SCN/BV-28-C
+LL/SEC/ADV/BV-02-C
+LL/SEC/ADV/BV-03-C
+LL/SEC/ADV/BV-04-C
+LL/SEC/ADV/BV-05-C
+LL/SEC/ADV/BV-06-C
+LL/SEC/ADV/BV-08-C
+LL/SEC/ADV/BV-09-C
+LL/SEC/ADV/BV-10-C
+LL/SEC/ADV/BV-11-C
+LL/SEC/ADV/BV-12-C
+LL/SEC/ADV/BV-13-C
+LL/SEC/ADV/BV-14-C
+LL/SEC/ADV/BV-15-C
+LL/SEC/ADV/BV-16-C
+LL/SEC/ADV/BV-17-C
+LL/SEC/ADV/BV-18-C
+LL/SEC/ADV/BV-20-C
+LL/SEC/SCN/BV-01-C