scripts: tests: sanitycheck: Add basic foundation for sanitycheck testsuite

This commit adds basic testcases for sanitycheck tool using pytest.
Coverage for the sanitycheck tool is obtained using coverage tool.
Instructions are included in the README.md in
scripts/tests/sanitycheck directory.

Signed-off-by: Aastha Grover <aastha.grover@intel.com>

CODEOWNERS

@@ -393,6 +393,7 @@
 /scripts/process_gperf.py                 @andrewboie
 /scripts/gen_relocate_app.py              @wentongwu
 /scripts/requirements*.txt                @mbolivar @galak @nashif
+/scripts/tests/sanitycheck/               @aasthagr
 /scripts/tracing/                         @wentongwu
 /scripts/sanity_chk/                      @nashif
 /scripts/sanitycheck                      @nashif

scripts/.gitignore

@@ -1,3 +1,7 @@
 parser.out
 parsetab.py
-
+tests/.mypy_cache
+tests/sanitycheck/.pytest_cache
+tests/sanitycheck/htmlcov
+tests/sanitycheck/__pycache__
+tests/sanitycheck/.coverage

scripts/requirements-build-test.txt

@@ -14,6 +14,7 @@ pykwalify
 
 # used for code coverage
 gcovr>=4.2
+coverage
 
 # used for west-command testing
 pytest

scripts/tests/sanitycheck/README.md

@@ -0,0 +1,47 @@
+# Sanitycheck Testing
+
+Running the tests require the environment variable ZEPHYR_BASE to be set.
+
+Sanitycheck Testsuite are located in $ZEPHYR_BASE/scripts/tests directory with all the data files in $ZEPHYR_BASE/scripts/test_data directory.
+
+## Dependencies
+
+Install all the dependencies using
+
+```
+pip install -r $ZEPHYR_BASE/scripts/tests/sanitycheck/requirements.txt
+```
+
+## Executing testsuite
+
+The testcases can be executed from the root directory using
+
+```
+pytest $ZEPHYR_BASE/scripts/tests/sanitycheck
+```
+
+## Sanitycheck Coverage
+
+The coverage for all the tests can be run using the command below. This will collect all the tests available.
+
+```bash
+coverage run -m pytest $ZEPHYR_BASE/scripts/tests/sanitycheck/
+```
+
+Then we can generate the coverage report for just sanitycheck script using
+
+```bash
+coverage report -m $ZEPHYR_BASE/scripts/sanitycheck
+```
+
+The html coverage report for sanitycheck can be generated using
+
+```bash
+coverage html sanitycheck
+```
+
+If needed,the full coverage html report can be generated in every run of "pytest" in the tests directory using configuration file (setup.cfg).
+
+## Organization of tests
+
+- test_sanitycheck.py : Contains basic testcases for environment variables, verifying testcase & platform schema's.

scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml

@@ -0,0 +1,6 @@
+tests:
+  logging.log_core:
+    tags: log_core logging
+    platform_exclude: nucleo_l053r8 nucleo_f030r8
+      stm32f0_disco native_posix native_posix_64 nrf52_bsim
+      qemu_riscv64

scripts/tests/sanitycheck/test_sanitycheck.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+"""
+This test file contains foundational testcases for Sanitycheck tool
+"""
+
+import os
+import imp
+import sys
+import pytest
+ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
+sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/"))
+from sanity_chk import scl
+sanitycheck = imp.load_source('sanitycheck', ZEPHYR_BASE + '/scripts/sanitycheck')
+
+@pytest.fixture(name='test_data')
+def _test_data():
+    """ Pytest fixture to set the path of test_data"""
+    data = ZEPHYR_BASE + "/scripts/tests/sanitycheck/test_data/"
+    return data
+
+def test_yamlload():
+    """ Test to check if loading the non-existent files raises the errors """
+    filename = 'testcase_nc.yaml'
+    with pytest.raises(FileNotFoundError):
+        scl.yaml_load(filename)
+
+def test_correct_testcase_schema(test_data):
+    """ Test to validate the testcase schema"""
+    filename = test_data + 'testcase_correct_schema.yaml'
+    schema = scl.yaml_load(ZEPHYR_BASE +'/scripts/sanity_chk/testcase-schema.yaml')
+    data = sanitycheck.SanityConfigParser(filename, schema)
+    data.load()
+    assert data