From e27cf15763385515a32e7fc1cf8e33ba22da07b0 Mon Sep 17 00:00:00 2001
From: Aastha Grover <aastha.grover@intel.com>
Date: Thu, 19 Mar 2020 15:27:27 -0700
Subject: [PATCH] scripts: tests: sanitycheck: Add basic foundation for
 sanitycheck testsuite

This commit adds basic testcases for sanitycheck tool using pytest.
Coverage for the sanitycheck tool is obtained using coverage tool.
Instructions are included in the README.md in
scripts/tests/sanitycheck directory.

Signed-off-by: Aastha Grover <aastha.grover@intel.com>
---
 CODEOWNERS                                    |  1 +
 scripts/.gitignore                            |  6 ++-
 scripts/requirements-build-test.txt           |  1 +
 scripts/tests/sanitycheck/README.md           | 47 +++++++++++++++++++
 .../test_data/testcase_correct_schema.yaml    |  6 +++
 scripts/tests/sanitycheck/test_sanitycheck.py | 36 ++++++++++++++
 6 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 scripts/tests/sanitycheck/README.md
 create mode 100644 scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml
 create mode 100644 scripts/tests/sanitycheck/test_sanitycheck.py

diff --git a/CODEOWNERS b/CODEOWNERS
index ea381be6b1f..6b0bd21c2cf 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -393,6 +393,7 @@
 /scripts/process_gperf.py                 @andrewboie
 /scripts/gen_relocate_app.py              @wentongwu
 /scripts/requirements*.txt                @mbolivar @galak @nashif
+/scripts/tests/sanitycheck/               @aasthagr
 /scripts/tracing/                         @wentongwu
 /scripts/sanity_chk/                      @nashif
 /scripts/sanitycheck                      @nashif
diff --git a/scripts/.gitignore b/scripts/.gitignore
index f38baba8439..5a488c27169 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -1,3 +1,7 @@
 parser.out
 parsetab.py
-
+tests/.mypy_cache
+tests/sanitycheck/.pytest_cache
+tests/sanitycheck/htmlcov
+tests/sanitycheck/__pycache__
+tests/sanitycheck/.coverage
diff --git a/scripts/requirements-build-test.txt b/scripts/requirements-build-test.txt
index cb3d68f354b..7326c1b674f 100644
--- a/scripts/requirements-build-test.txt
+++ b/scripts/requirements-build-test.txt
@@ -14,6 +14,7 @@ pykwalify
 
 # used for code coverage
 gcovr>=4.2
+coverage
 
 # used for west-command testing
 pytest
diff --git a/scripts/tests/sanitycheck/README.md b/scripts/tests/sanitycheck/README.md
new file mode 100644
index 00000000000..34eea3b1240
--- /dev/null
+++ b/scripts/tests/sanitycheck/README.md
@@ -0,0 +1,47 @@
+# Sanitycheck Testing
+
+Running the tests require the environment variable ZEPHYR_BASE to be set.
+
+Sanitycheck Testsuite are located in $ZEPHYR_BASE/scripts/tests directory with all the data files in $ZEPHYR_BASE/scripts/test_data directory.
+
+## Dependencies
+
+Install all the dependencies using
+
+```
+pip install -r $ZEPHYR_BASE/scripts/tests/sanitycheck/requirements.txt
+```
+
+## Executing testsuite
+
+The testcases can be executed from the root directory using
+
+```
+pytest $ZEPHYR_BASE/scripts/tests/sanitycheck
+```
+
+## Sanitycheck Coverage
+
+The coverage for all the tests can be run using the command below. This will collect all the tests available.
+
+```bash
+coverage run -m pytest $ZEPHYR_BASE/scripts/tests/sanitycheck/
+```
+
+Then we can generate the coverage report for just sanitycheck script using
+
+```bash
+coverage report -m $ZEPHYR_BASE/scripts/sanitycheck
+```
+
+The html coverage report for sanitycheck can be generated using
+
+```bash
+coverage html sanitycheck
+```
+
+If needed,the full coverage html report can be generated in every run of "pytest" in the tests directory using configuration file (setup.cfg).
+
+## Organization of tests
+
+- test_sanitycheck.py : Contains basic testcases for environment variables, verifying testcase & platform schema's.
diff --git a/scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml b/scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml
new file mode 100644
index 00000000000..c44e3ec4327
--- /dev/null
+++ b/scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml
@@ -0,0 +1,6 @@
+tests:
+  logging.log_core:
+    tags: log_core logging
+    platform_exclude: nucleo_l053r8 nucleo_f030r8
+      stm32f0_disco native_posix native_posix_64 nrf52_bsim
+      qemu_riscv64
diff --git a/scripts/tests/sanitycheck/test_sanitycheck.py b/scripts/tests/sanitycheck/test_sanitycheck.py
new file mode 100644
index 00000000000..910194e93bb
--- /dev/null
+++ b/scripts/tests/sanitycheck/test_sanitycheck.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+"""
+This test file contains foundational testcases for Sanitycheck tool
+"""
+
+import os
+import imp
+import sys
+import pytest
+ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
+sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/"))
+from sanity_chk import scl
+sanitycheck = imp.load_source('sanitycheck', ZEPHYR_BASE + '/scripts/sanitycheck')
+
+@pytest.fixture(name='test_data')
+def _test_data():
+    """ Pytest fixture to set the path of test_data"""
+    data = ZEPHYR_BASE + "/scripts/tests/sanitycheck/test_data/"
+    return data
+
+def test_yamlload():
+    """ Test to check if loading the non-existent files raises the errors """
+    filename = 'testcase_nc.yaml'
+    with pytest.raises(FileNotFoundError):
+        scl.yaml_load(filename)
+
+def test_correct_testcase_schema(test_data):
+    """ Test to validate the testcase schema"""
+    filename = test_data + 'testcase_correct_schema.yaml'
+    schema = scl.yaml_load(ZEPHYR_BASE +'/scripts/sanity_chk/testcase-schema.yaml')
+    data = sanitycheck.SanityConfigParser(filename, schema)
+    data.load()
+    assert data