diff --git a/doc/guides/test/sanitycheck.rst b/doc/guides/test/sanitycheck.rst index 5e447eb58d7..91601b637e6 100644 --- a/doc/guides/test/sanitycheck.rst +++ b/doc/guides/test/sanitycheck.rst @@ -19,6 +19,21 @@ environment, but it does sufficient testing by building samples and tests for different boards and different configurations to help keep the full code tree buildable. +When using (at least) one ``-v`` option, sanitycheck's console output +shows for every test how the test is run (qemu, native_posix, etc.) or +whether the image is just built. There are a few reasons why sanitycheck +only builds a test and doesn't run it: + +- The test is marked as ``build_only: true`` in its ``.yaml`` + configuration file. +- The test configuration has defined a ``harness`` but you don't have + it or haven't set it up. +- You or some higher level automation invoked sanitycheck with + ``--build-only``. + +These also affect the outputs of ``--testcase-report`` and +``--detailed-report``, see their respective ``--help`` sections. + To run the script in the local tree, follow the steps below: :: @@ -38,7 +53,8 @@ a simulated (QEMU) environment. The sanitycheck script accepts the following optional arguments: - -h, --help show this help message and exit + -h, --help Show the complete and most up-to-date help message + and exit. -p PLATFORM, --platform PLATFORM Platform filter for testing. This option may be used multiple times. Testcases will only be built/run on @@ -349,7 +365,7 @@ extra_configs: build_only: (default False) - If true, don't try to run the test under QEMU even if the + If true, don't try to run the test even if the selected platform supports it. build_on_all: (default False) diff --git a/scripts/sanitycheck b/scripts/sanitycheck index 019ab073c51..5fce4b42b26 100755 --- a/scripts/sanitycheck +++ b/scripts/sanitycheck @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 """Zephyr Sanity Tests +Also check the "User and Developer Guides" at https://docs.zephyrproject.org/ + This script scans for the set of unit test applications in the git repository and attempts to execute them. By default, it tries to build each test case on one platform per architecture, using a precedence @@ -42,8 +44,8 @@ pairs: when building or running the test case. build_only: (default False) - If true, don't try to run the test under QEMU even if the - selected platform supports it. + If true, don't try to run the test even if the selected platform + supports it. build_on_all: (default False) If true, attempt to build test on all available platforms. @@ -2723,8 +2725,8 @@ def parse_arguments(): """ Artificially long but functional example: $ ./scripts/sanitycheck -v \\ - --testcase-root tests/ \\ - --testcase-root mytests/ \\ + --testcase-root tests/ztest/base \\ + --testcase-root tests/kernel \\ --test tests/ztest/base/testing.ztest.verbose_0 \\ --test tests/kernel/fifo/fifo_api/kernel.fifo.poll @@ -2776,7 +2778,7 @@ Artificially long but functional example: test_xor_subtest.add_argument( "-s", "--test", action="append", help="Run only the specified test cases. These are named by " - "path/relative/to/Zephyr/base/section.name.in.testcase.yaml>") + "") test_xor_subtest.add_argument( "--sub-test", action="append", @@ -2794,7 +2796,9 @@ Artificially long but functional example: parser.add_argument( "-o", "--testcase-report", - help="Output a CSV spreadsheet containing results of the test run") + help="""Output a CSV spreadsheet containing results of the test run. + The handler_time column is left blank for tests that were only + compiled and not run.""") parser.add_argument( "-d", "--discard-report", help="Output a CSV spreadsheet showing tests that were skipped " @@ -2844,7 +2848,11 @@ Artificially long but functional example: parser.add_argument("--detailed-report", action="store", metavar="FILENAME", - help="Generate a junit report with detailed testcase results.") + help="""Generate a junit report with detailed testcase results. + Unlike the CSV file produced by --testcase-report, this XML + report includes only tests which have run and none which were + merely built. If an image with multiple tests crashes early then + later tests are not accounted for either.""") parser.add_argument("--timestamps", action="store_true",