snippets: initial snippet.yml support

Add a new script, snippets.py, which is responsible for searching
SNIPPET_ROOT for snippet definitions, validating them, and informing
the build system about what needs doing as a result.

Use this script in snippets.cmake to:

- validate any discovered snippet.yml files
- error out on undefined snippets
- add a 'snippets' build system target that prints all snippet
  names (analogous to 'boards' and 'shields' targets)
- handle any specific build system settings properly,
  by include()-ing a file it generates

With this patch, you can define or extend a snippet in a snippet.yml
file anywhere underneath a directory in SNIPPET_ROOT. The snippet.yml
file format has a schema whose initial definition is in a new file,
snippet-schema.yml.

This initial snippet.yml file format supports adding .overlay and
.conf files, like this:

  name: foo
  append:
    DTC_OVERLAY_FILE: foo.overlay
    OVERLAY_CONFIG: foo.conf
  boards:
    myboard:
      append:
        DTC_OVERLAY_FILE: myboard.overlay
        OVERLAY_CONFIG: myboard.conf
    /my-regular-expression-over-board-names/:
      append:
        DTC_OVERLAY_FILE: myregexp.overlay
        OVERLAY_CONFIG: myregexp.conf

(Note that since the snippet feature is intended to be extensible, the
same snippet name may appear in multiple files throughout any
directory in SNIPPET_ROOT, with each addition augmenting prior ones.)

This initial syntax aligns with the following snippet design goals:

- extensible: you can add board-specific support for an existing
  snippet in another module

- able to combine multiple types of configuration: we can now apply a
  .overlay and .conf at the same time

- specializable: this allows you to define settings that only apply
  to a selectable set of boards (including with regular expression
  support for matching against multiple similar boards that follow
  a naming convention)

- DRY: you can use regular expressions to apply the same snippet
  settings to multiple boards like this: /(board1|board2|...)/

This patch is not trying to design and implement everything up front.
Additional features can and will be added to the snippet.yml format
over time; using YAML as a format allows us to make
backwards-compatible extensions as needed.

Signed-off-by: Marti Bolivar <marti.bolivar@nordicsemi.no>

cmake/modules/snippets.cmake

@@ -4,11 +4,33 @@
 
 # Snippets support
 #
+# This module:
+#
+# - searches for snippets in zephyr and any modules
+# - validates the SNIPPET input variable, if any
+#
+# If SNIPPET contains a snippet name that is not found, an error
+# will be raised, and the list of valid snippets will be printed.
+#
 # Outcome:
 # The following variables will be defined when this module completes:
 #
 # - SNIPPET_AS_LIST: CMake list of snippet names, created from the
 #   SNIPPET variable
+# - SNIPPET_ROOT: CMake list of snippet roots, deduplicated and with
+#   ZEPHYR_BASE appended at the end
+#
+# The following variables may be updated when this module completes:
+# - DTC_OVERLAY_FILE
+# - OVERLAY_CONFIG
+#
+# The following targets will be defined when this CMake module completes:
+# - snippets: when invoked, a list of valid snippets will be printed
+#
+# Optional variables:
+# - SNIPPET_ROOT: input CMake list of snippet roots (directories containing
+#   additional snippet implementations); this should not include ZEPHYR_BASE,
+#   as that will be added by this module
 
 include_guard(GLOBAL)
 
@@ -21,15 +43,83 @@ zephyr_check_cache(SNIPPET WATCH)
 # parent scope. We'll set our outcome variables in the parent scope of
 # the function to ensure the outcome of the module.
 function(zephyr_process_snippets)
-  if (SNIPPET)
-    message(STATUS "Snippet(s): ${SNIPPET}")
-  else()
+  set(snippets_py ${ZEPHYR_BASE}/scripts/snippets.py)
+  set(snippets_generated ${CMAKE_BINARY_DIR}/zephyr/snippets_generated.cmake)
+
+  # Set SNIPPET_AS_LIST, removing snippets_generated.cmake if we are
+  # running cmake again and snippets are no longer requested.
+  if (NOT DEFINED SNIPPET)
     set(SNIPPET_AS_LIST "" PARENT_SCOPE)
-    return()
+    file(REMOVE ${snippets_generated})
+  else()
+    string(REPLACE " " ";" SNIPPET_AS_LIST "${SNIPPET}")
+    set(SNIPPET_AS_LIST "${SNIPPET_AS_LIST}" PARENT_SCOPE)
   endif()
 
-  string(REPLACE " " ";" SNIPPET_AS_LIST "${SNIPPET}")
-  set(SNIPPET_AS_LIST "${SNIPPET_AS_LIST}" PARENT_SCOPE)
+  # Set SNIPPET_ROOT.
+  list(APPEND SNIPPET_ROOT ${APPLICATION_SOURCE_DIR})
+  list(APPEND SNIPPET_ROOT ${ZEPHYR_BASE})
+  unset(real_snippet_root)
+  foreach(snippet_dir ${SNIPPET_ROOT})
+    # The user might have put a symbolic link in here, for example.
+    file(REAL_PATH ${snippet_dir} real_snippet_dir)
+    list(APPEND real_snippet_root ${real_snippet_dir})
+  endforeach()
+  set(SNIPPET_ROOT ${real_snippet_root})
+  list(REMOVE_DUPLICATES SNIPPET_ROOT)
+  set(SNIPPET_ROOT "${SNIPPET_ROOT}" PARENT_SCOPE)
+
+  # Generate and include snippets_generated.cmake.
+  # The Python script is responsible for checking for unknown
+  # snippets.
+  set(snippet_root_args)
+  foreach(root IN LISTS SNIPPET_ROOT)
+    list(APPEND snippet_root_args --snippet-root "${root}")
+  endforeach()
+  set(requested_snippet_args)
+  foreach(snippet_name ${SNIPPET_AS_LIST})
+    list(APPEND requested_snippet_args --snippet "${snippet_name}")
+  endforeach()
+  execute_process(COMMAND ${PYTHON_EXECUTABLE}
+    ${snippets_py}
+    ${snippet_root_args}
+    ${requested_snippet_args}
+    --cmake-out ${snippets_generated}
+    OUTPUT_VARIABLE output
+    ERROR_VARIABLE output
+    RESULT_VARIABLE ret)
+  if(${ret})
+    message(FATAL_ERROR "${output}")
+  endif()
+  include(${snippets_generated})
+
+  # Propagate include()d build system settings to the caller.
+  set(DTC_OVERLAY_FILE ${DTC_OVERLAY_FILE} PARENT_SCOPE)
+  set(OVERLAY_CONFIG ${OVERLAY_CONFIG} PARENT_SCOPE)
+
+  # Create the 'snippets' target. Each snippet is printed in a
+  # separate command because build system files are not fond of
+  # newlines.
+  list(TRANSFORM SNIPPET_NAMES PREPEND "COMMAND;${CMAKE_COMMAND};-E;echo;"
+    OUTPUT_VARIABLE snippets_target_cmd)
+  add_custom_target(snippets ${snippets_target_cmd} USES_TERMINAL)
+
+  # If snippets were requested, print messages for each one.
+  if(SNIPPET_AS_LIST)
+    # Print the requested snippets.
+    set(snippet_names "Snippet(s):")
+    foreach(snippet IN LISTS SNIPPET_AS_LIST)
+      string(APPEND snippet_names " ${snippet}")
+    endforeach()
+    message(STATUS "${snippet_names}")
+  endif()
+
+  # Re-run cmake if any files we depend on changed.
+  set_property(DIRECTORY APPEND PROPERTY
+    CMAKE_CONFIGURE_DEPENDS
+    ${snippets_py}
+    ${SNIPPET_PATHS}            #  generated variable
+    )
 endfunction()
 
 zephyr_process_snippets()

scripts/schemas/snippet-schema.yml

@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (c) 2022, Nordic Semiconductor ASA
+
+# A pykwalify schema for basic validation of the snippet.yml format.
+
+schema;append-schema:
+  # Sub-schema for appending onto CMake list variables.
+  # See uses under 'append:' keys below.
+  type: map
+  mapping:
+    DTC_OVERLAY_FILE:
+      type: str
+    OVERLAY_CONFIG:
+      type: str
+
+type: map
+mapping:
+  name:
+    required: true
+    type: str
+  append:
+    example: |
+      Snippet-wide appending can be done here:
+
+        name: foo
+        append:
+          DTC_OVERLAY_FILE: m3.overlay
+    include: append-schema
+  boards:
+    example: |
+      Board-specific appending can be done here:
+
+        name: foo
+        boards:
+          qemu_cortex_m3:
+            append:
+              DTC_OVERLAY_FILE: m3.overlay
+    type: map
+    mapping:
+      regex;(.*):
+        type: map
+        mapping:
+          append:
+            include: append-schema

scripts/snippets.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022, Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Internal snippets tool.
+This is part of the build system's support for snippets.
+It is not meant for use outside of the build system.
+
+Output CMake variables:
+
+- SNIPPET_NAMES: CMake list of discovered snippet names
+- SNIPPET_FOUND_{snippet}: one per discovered snippet
+'''
+
+from collections import defaultdict, UserDict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Iterable, List, Set
+import argparse
+import logging
+import os
+import pykwalify.core
+import pykwalify.errors
+import re
+import sys
+import textwrap
+import yaml
+
+# Marker type for an 'append:' configuration. Maps variables
+# to the list of values to append to them.
+Appends = Dict[str, List[str]]
+
+def _new_append():
+    return defaultdict(list)
+
+def _new_board2appends():
+    return defaultdict(_new_append)
+
+@dataclass
+class Snippet:
+    '''Class for keeping track of all the settings discovered for an
+    individual snippet.'''
+
+    name: str
+    appends: Appends = field(default_factory=_new_append)
+    board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)
+
+    def process_data(self, pathobj: Path, snippet_data: dict):
+        '''Process the data in a snippet.yml file, after it is loaded into a
+        python object and validated by pykwalify.'''
+        def append_value(variable, value):
+            if variable in ('DTC_OVERLAY_FILE', 'OVERLAY_CONFIG'):
+                path = pathobj.parent / value
+                if not path.is_file():
+                    _err(f'snippet file {pathobj}: {variable}: file not found: {path}')
+                return f'"{path}"'
+            _err(f'unknown append variable: {variable}')
+
+        for variable, value in snippet_data.get('append', {}).items():
+            self.appends[variable].append(append_value(variable, value))
+        for board, settings in snippet_data.get('boards', {}).items():
+            if board.startswith('/') and not board.endswith('/'):
+                _err(f"snippet file {pathobj}: board {board} starts with '/', so "
+                     "it must end with '/' to use a regular expression")
+            for variable, value in settings.get('append', {}).items():
+                self.board2appends[board][variable].append(
+                    append_value(variable, value))
+
+class Snippets(UserDict):
+    '''Type for all the information we have discovered about all snippets.
+    As a dict, this maps a snippet's name onto the Snippet object.
+    Any additional global attributes about all snippets go here as
+    instance attributes.'''
+
+    def __init__(self, requested: Iterable[str] = None):
+        super().__init__()
+        self.paths: Set[Path] = set()
+        self.requested: Set[str] = set(requested or [])
+
+class SnippetsError(Exception):
+    '''Class for signalling expected errors'''
+
+    def __init__(self, msg):
+        self.msg = msg
+
+class SnippetToCMakePrinter:
+    '''Helper class for printing a Snippets's semantics to a .cmake
+    include file for use by snippets.cmake.'''
+
+    def __init__(self, snippets: Snippets, out_file):
+        self.snippets = snippets
+        self.out_file = out_file
+        self.section = '#' * 79
+
+    def print_cmake(self):
+        '''Print to the output file provided to the constructor.'''
+        # TODO: add source file info
+        snippets = self.snippets
+        snippet_names = sorted(snippets.keys())
+        snippet_path_list = " ".join(
+            sorted(f'"{path}"' for path in snippets.paths))
+
+        self.print('''\
+# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
+#
+# This file contains build system settings derived from your snippets.
+# Its contents are an implementation detail that should not be used outside
+# of Zephyr's snippets CMake module.
+#
+# See the Snippets guide in the Zephyr documentation for more information.
+''')
+
+        self.print(f'''\
+{self.section}
+# Global information about all snippets.
+
+# The name of every snippet that was discovered.
+set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
+# The paths to all the snippet.yml files. One snippet
+# can have multiple snippet.yml files.
+set(SNIPPET_PATHS {snippet_path_list})
+''')
+
+        for snippet_name in snippet_names:
+            if snippet_name not in snippets.requested:
+                continue
+            self.print_cmake_for(snippets[snippet_name])
+            self.print()
+
+    def print_cmake_for(self, snippet: Snippet):
+        self.print(f'''\
+{self.section}
+# Snippet '{snippet.name}'
+
+# Common variable appends.''')
+        self.print_appends(snippet.appends, 0)
+        for board, appends in snippet.board2appends.items():
+            self.print_appends_for_board(board, appends)
+
+    def print_appends_for_board(self, board: str, appends: Appends):
+        if board.startswith('/'):
+            board_re = board[1:-1]
+            self.print(f'''\
+# Appends for board regular expression '{board_re}'
+if("${{BOARD}}" MATCHES "^{board_re}$")''')
+        else:
+            self.print(f'''\
+# Appends for board '{board}'
+if("${{BOARD}}" STREQUAL "{board}")''')
+        self.print_appends(appends, 1)
+        self.print('endif()')
+
+    def print_appends(self, appends: Appends, indent: int):
+        space = '  ' * indent
+        for name, values in appends.items():
+            for value in values:
+                self.print(f'{space}list(APPEND {name} {value})')
+
+    def print(self, *args, **kwargs):
+        kwargs['file'] = self.out_file
+        print(*args, **kwargs)
+
+# Name of the file containing the pykwalify schema for snippet.yml
+# files.
+SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
+with open(SCHEMA_PATH, 'r') as f:
+    SNIPPET_SCHEMA = yaml.safe_load(f.read())
+
+# The name of the file which contains metadata about the snippets
+# being defined in a directory.
+SNIPPET_YML = 'snippet.yml'
+
+# Regular expression for validating snippet names. Snippet names must
+# begin with an alphanumeric character, and may contain alphanumeric
+# characters or underscores. This is intentionally very restrictive to
+# keep things consistent and easy to type and remember. We can relax
+# this a bit later if needed.
+SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')
+
+# Logger for this module.
+LOG = logging.getLogger('snippets')
+
+def _err(msg):
+    raise SnippetsError(f'error: {msg}')
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='snippets helper',
+                                     allow_abbrev=False)
+    parser.add_argument('--snippet-root', default=[], action='append', type=Path,
+                        help='''a SNIPPET_ROOT element; may be given
+                        multiple times''')
+    parser.add_argument('--snippet', dest='snippets', default=[], action='append',
+                        help='''a SNIPPET element; may be given
+                        multiple times''')
+    parser.add_argument('--cmake-out', type=Path,
+                        help='''file to write cmake output to; include()
+                        this file after calling this script''')
+    return parser.parse_args()
+
+def setup_logging():
+    # Silence validation errors from pykwalify, which are logged at
+    # logging.ERROR level. We want to handle those ourselves as
+    # needed.
+    logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
+    logging.basicConfig(level=logging.INFO,
+                        format='  %(name)s: %(message)s')
+
+def process_snippets(args: argparse.Namespace) -> Snippets:
+    '''Process snippet.yml files under each *snippet_root*
+    by recursive search. Return a Snippets object describing
+    the results of the search.
+    '''
+    # This will contain information about all the snippets
+    # we discover in each snippet_root element.
+    snippets = Snippets(requested=args.snippets)
+
+    # Process each path in snippet_root in order, adjusting
+    # snippets as needed for each one.
+    for root in args.snippet_root:
+        process_snippets_in(root, snippets)
+
+    return snippets
+
+def process_snippets_in(root_dir: Path, snippets: Snippets) -> None:
+    '''Process snippet.yml files in *root_dir*,
+    updating *snippets* as needed.'''
+
+    if not root_dir.is_dir():
+        LOG.warning(f'SNIPPET_ROOT {root_dir} '
+                    'is not a directory; ignoring it')
+        return
+
+    snippets_dir = root_dir / 'snippets'
+    if not snippets_dir.is_dir():
+        return
+
+    for dirpath, _, filenames in os.walk(snippets_dir):
+        if SNIPPET_YML not in filenames:
+            continue
+
+        snippet_yml = Path(dirpath) / SNIPPET_YML
+        snippet_data = load_snippet_yml(snippet_yml)
+        name = snippet_data['name']
+        if name not in snippets:
+            snippets[name] = Snippet(name=name)
+        snippets[name].process_data(snippet_yml, snippet_data)
+        snippets.paths.add(snippet_yml)
+
+def load_snippet_yml(snippet_yml: Path) -> dict:
+    '''Load a snippet.yml file *snippet_yml*, validate the contents
+    against the schema, and do other basic checks. Return the dict
+    of the resulting YAML data.'''
+
+    with open(snippet_yml, 'r') as f:
+        try:
+            snippet_data = yaml.safe_load(f.read())
+        except yaml.scanner.ScannerError:
+            _err(f'snippets file {snippet_yml} is invalid YAML')
+
+    def pykwalify_err(e):
+        return f'''\
+invalid {SNIPPET_YML} file: {snippet_yml}
+{textwrap.indent(e.msg, '  ')}
+'''
+
+    try:
+        pykwalify.core.Core(source_data=snippet_data,
+                            schema_data=SNIPPET_SCHEMA).validate()
+    except pykwalify.errors.PyKwalifyException as e:
+        _err(pykwalify_err(e))
+
+    name = snippet_data['name']
+    if not SNIPPET_NAME_RE.fullmatch(name):
+        _err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
+             'snippet names must begin with a letter '
+             'or number, and may only contain letters, numbers, '
+             'dashes (-), and underscores (_)')
+
+    return snippet_data
+
+def check_for_errors(snippets: Snippets) -> None:
+    unknown_snippets = sorted(snippet for snippet in snippets.requested
+                              if snippet not in snippets)
+    if unknown_snippets:
+        all_snippets = '\n    '.join(sorted(snippets))
+        _err(f'''\
+snippets not found: {', '.join(unknown_snippets)}
+  Please choose from among the following snippets:
+    {all_snippets}''')
+
+def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
+    '''Write a cmake include file to *cmake_out* which
+    reflects the information in *snippets*.
+
+    The contents of this file should be considered an implementation
+    detail and are not meant to be used outside of snippets.cmake.'''
+    if not cmake_out.parent.exists():
+        cmake_out.parent.mkdir()
+    with open(cmake_out, 'w') as f:
+        SnippetToCMakePrinter(snippets, f).print_cmake()
+
+def main():
+    args = parse_args()
+    setup_logging()
+    try:
+        snippets = process_snippets(args)
+        check_for_errors(snippets)
+    except SnippetsError as e:
+        LOG.critical(e.msg)
+        sys.exit(1)
+    write_cmake_out(snippets, args.cmake_out)
+
+if __name__ == "__main__":
+    main()