sanitycheck: add testcases for add_testcases function of testsuite class

test_testsuite_class.py: Add testcases for add_testcases function
of testsuite class in sanitycheck.
test_data/testcases/tests & /samples : Testcase root directory
to add all the testcases & to test add_testcases function.
conftest.py: Module for common pytest fixtures, also used for
passing data from one testcase to another.
Note: conftest.py has a class_testsuite fixture where board_root is
defined as the directory which will be added in a separate PR.

Signed-off-by: Aastha Grover <aastha.grover@intel.com>

scripts/tests/sanitycheck/conftest.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+# pylint: disable=redefined-outer-name
+# pylint: disable=line-too-long
+'''Common fixtures for use in testing the sanitycheck tool.'''
+
+import os
+import sys
+import pytest
+
+ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
+sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/sanity_chk"))
+from sanitylib import TestSuite
+
+@pytest.fixture(name='test_data')
+def _test_data():
+    """ Pytest fixture to load the test data directory"""
+    data = ZEPHYR_BASE + "/scripts/tests/sanitycheck/test_data/"
+    return data
+
+@pytest.fixture
+def testcases_dir():
+    """ Pytest fixture to load the test data directory"""
+    return ZEPHYR_BASE + "/scripts/tests/sanitycheck/test_data/testcases"
+
+@pytest.fixture
+def class_testsuite(test_data, testcases_dir):
+    """ Pytest fixture to initialize and return the class TestSuite object"""
+    board_root = test_data +"board_config/1_level/2_level/"
+    testcase_root = [testcases_dir + '/tests', testcases_dir + '/samples']
+    outdir = test_data +'sanity_out_demo'
+    suite = TestSuite(board_root, testcase_root, outdir)
+    return suite

scripts/tests/sanitycheck/test_data/testcase_correct_schema.yaml

@@ -1,6 +1,4 @@
 tests:
-  logging.log_core:
-    tags: log_core logging
-    platform_exclude: nucleo_l053r8 nucleo_f030r8
-      stm32f0_disco native_posix native_posix_64 nrf52_bsim
-      qemu_riscv64
+  testing.correct_schema:
+    tags: demo_correct correct_tags
+    platform_exclude: demo_board_1

scripts/tests/sanitycheck/test_data/testcases/samples/test_app/test_sample_app.yaml

@@ -0,0 +1,11 @@
+sample:
+  name: Sample Library
+tests:
+  sample_test.app:
+    tags: sample_tag
+    harness: console
+    harness_config:
+      type: multi_line
+      regex:
+        - "Hello World!"
+        - "Sample says: Hello World!"

scripts/tests/sanitycheck/test_data/testcases/tests/test_a/test_data.yaml

@@ -0,0 +1,6 @@
+tests:
+  test_a.check_1:
+    tags: test_a
+  test_a.check_2:
+    extra_args: CONF_FILE="test.conf"
+    tags: test_a

scripts/tests/sanitycheck/test_data/testcases/tests/test_b/test_data.yaml

@@ -0,0 +1,8 @@
+tests:
+  test_b.check_1:
+    min_ram: 32
+    tags: test_b
+  test_b.check_2:
+    min_ram: 32
+    extra_args: CONF_FILE="test.conf"
+    tags: test_b

scripts/tests/sanitycheck/test_data/testcases/tests/test_c/test_data.yaml

@@ -0,0 +1,6 @@
+tests:
+  test_c.check_1:
+    tags: test_c
+  test_c.check_2:
+    extra_args: CONF_FILE="test.conf"
+    tags: test_c

scripts/tests/sanitycheck/test_sanitycheck.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2020 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-"""
-This test file contains foundational testcases for Sanitycheck tool
-"""
-
-import os
-import sys
-import pytest
-
-ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
-sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/sanity_chk"))
-
-import scl
-from sanitylib import SanityConfigParser
-
-@pytest.fixture(name='test_data')
-def _test_data():
-    """ Pytest fixture to set the path of test_data"""
-    data = ZEPHYR_BASE + "/scripts/tests/sanitycheck/test_data/"
-    return data
-
-def test_yamlload():
-    """ Test to check if loading the non-existent files raises the errors """
-    filename = 'testcase_nc.yaml'
-    with pytest.raises(FileNotFoundError):
-        scl.yaml_load(filename)
-
-def test_correct_testcase_schema(test_data):
-    """ Test to validate the testcase schema"""
-    filename = test_data + 'testcase_correct_schema.yaml'
-    schema = scl.yaml_load(ZEPHYR_BASE +'/scripts/sanity_chk/testcase-schema.yaml')
-    data = SanityConfigParser(filename, schema)
-    data.load()
-    assert data

scripts/tests/sanitycheck/test_testsuite_class.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+# pylint: disable=line-too-long
+# pylint: disable=C0321
+'''
+This test file contains testcases for Testsuite class of sanitycheck
+'''
+import sys
+import os
+
+ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
+sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/sanity_chk"))
+
+from sanitylib import TestCase
+
+def test_testsuite_add_testcases(class_testsuite):
+    """ Testing add_testcase function of Testsuite class in sanitycheck """
+    # Test 1: Check the list of testcases after calling add testcases function is as expected
+    class_testsuite.SAMPLE_FILENAME = 'test_sample_app.yaml'
+    class_testsuite.TESTCASE_FILENAME = 'test_data.yaml'
+    class_testsuite.add_testcases()
+    tests_rel_dir = 'scripts/tests/sanitycheck/test_data/testcases/tests/'
+    expected_testcases = ['test_b.check_1',
+                          'test_b.check_2',
+                          'test_c.check_1',
+                          'test_c.check_2',
+                          'test_a.check_1',
+                          'test_a.check_2',
+                          'sample_test.app']
+    testcase_list = []
+    for key in sorted(class_testsuite.testcases.keys()):
+        testcase_list.append(os.path.basename(os.path.normpath(key)))
+    assert sorted(testcase_list) == sorted(expected_testcases)
+
+    # Test 2 : Assert Testcase name is expected & all the testcases values are testcase class objects
+    testcase = class_testsuite.testcases.get(tests_rel_dir + 'test_a/test_a.check_1')
+    assert testcase.name == tests_rel_dir + 'test_a/test_a.check_1'
+    assert all(isinstance(n, TestCase) for n in class_testsuite.testcases.values())