diff --git a/CODEOWNERS b/CODEOWNERS index ea381be6b1f..6b0bd21c2cf 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -393,6 +393,7 @@ /scripts/process_gperf.py @andrewboie /scripts/gen_relocate_app.py @wentongwu /scripts/requirements*.txt @mbolivar @galak @nashif +/scripts/tests/sanitycheck/ @aasthagr /scripts/tracing/ @wentongwu /scripts/sanity_chk/ @nashif /scripts/sanitycheck @nashif diff --git a/scripts/.gitignore b/scripts/.gitignore index f38baba8439..5a488c27169 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,3 +1,7 @@ parser.out parsetab.py - +tests/.mypy_cache +tests/sanitycheck/.pytest_cache +tests/sanitycheck/htmlcov +tests/sanitycheck/__pycache__ +tests/sanitycheck/.coverage diff --git a/scripts/requirements-build-test.txt b/scripts/requirements-build-test.txt index cb3d68f354b..7326c1b674f 100644 --- a/scripts/requirements-build-test.txt +++ b/scripts/requirements-build-test.txt @@ -14,6 +14,7 @@ pykwalify # used for code coverage gcovr>=4.2 +coverage # used for west-command testing pytest diff --git a/scripts/tests/sanitycheck/README.md b/scripts/tests/sanitycheck/README.md new file mode 100644 index 00000000000..34eea3b1240 --- /dev/null +++ b/scripts/tests/sanitycheck/README.md @@ -0,0 +1,47 @@ +# Sanitycheck Testing + +Running the tests require the environment variable ZEPHYR_BASE to be set. + +Sanitycheck Testsuite are located in $ZEPHYR_BASE/scripts/tests directory with all the data files in $ZEPHYR_BASE/scripts/test_data directory. + +## Dependencies + +Install all the dependencies using + +``` +pip install -r $ZEPHYR_BASE/scripts/tests/sanitycheck/requirements.txt +``` + +## Executing testsuite + +The testcases can be executed from the root directory using + +``` +pytest $ZEPHYR_BASE/scripts/tests/sanitycheck +``` + +## Sanitycheck Coverage + +The coverage for all the tests can be run using the command below. This will collect all the tests available. + +```bash +coverage run -m pytest $ZEPHYR_BASE/scripts/tests/sanitycheck/ +``` + +Then we can generate the coverage report for just sanitycheck script using + +```bash +coverage report -m $ZEPHYR_BASE/scripts/sanitycheck +``` + +The html coverage report for sanitycheck can be generated using + +```bash +coverage html sanitycheck +``` + +If needed,the full coverage html report can be generated in every run of "pytest" in the tests directory using configuration file (setup.cfg). + +## Organization of tests + +- test_sanitycheck.py : Contains basic testcases for environment variables, verifying testcase & platform schema's. diff --git a/scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml b/scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml new file mode 100644 index 00000000000..c44e3ec4327 --- /dev/null +++ b/scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml @@ -0,0 +1,6 @@ +tests: + logging.log_core: + tags: log_core logging + platform_exclude: nucleo_l053r8 nucleo_f030r8 + stm32f0_disco native_posix native_posix_64 nrf52_bsim + qemu_riscv64 diff --git a/scripts/tests/sanitycheck/test_sanitycheck.py b/scripts/tests/sanitycheck/test_sanitycheck.py new file mode 100644 index 00000000000..910194e93bb --- /dev/null +++ b/scripts/tests/sanitycheck/test_sanitycheck.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +""" +This test file contains foundational testcases for Sanitycheck tool +""" + +import os +import imp +import sys +import pytest +ZEPHYR_BASE = os.getenv("ZEPHYR_BASE") +sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/")) +from sanity_chk import scl +sanitycheck = imp.load_source('sanitycheck', ZEPHYR_BASE + '/scripts/sanitycheck') + +@pytest.fixture(name='test_data') +def _test_data(): + """ Pytest fixture to set the path of test_data""" + data = ZEPHYR_BASE + "/scripts/tests/sanitycheck/test_data/" + return data + +def test_yamlload(): + """ Test to check if loading the non-existent files raises the errors """ + filename = 'testcase_nc.yaml' + with pytest.raises(FileNotFoundError): + scl.yaml_load(filename) + +def test_correct_testcase_schema(test_data): + """ Test to validate the testcase schema""" + filename = test_data + 'testcase_correct_schema.yaml' + schema = scl.yaml_load(ZEPHYR_BASE +'/scripts/sanity_chk/testcase-schema.yaml') + data = sanitycheck.SanityConfigParser(filename, schema) + data.load() + assert data