cmake: re-work devicetree preprocessing steps

This causes any devicetree error messages to refer to the source files
which contain the errors instead of intermediate <BOARD>.dts.pre.tmp
files in the build directory.

Do this using a new python-devicetree feature which can correctly
handle cpp's generated line marker directives.

To use this feature, rework the way that the C preprocessor is run on
devicetree inputs so that line marker directives are preserved by
removing the -P option.

This is a backwards incompatible change to the way the generated files
in the build directory used to work, as not all tools can consume line
markers. In particular, dtc can't handle these lines. We therefore
pass dtc zephyr.dts instead (the final parsed devicetree output from
python-devicetree).

Since <BOARD>.dts.pre.tmp is a publicly documented file, this is
changing existing behavior, so use a new file name for the
intermediate files to make it more obvious that something changed. In
particular, use zephyr.dts.pre instead of <BOARD>.dts.pre.tmp.
(The $BOARD.dts.pre.tmp name is a little cumbersome anyway.)

Signed-off-by: Martí Bolívar <marti.bolivar@nordicsemi.no>

cmake/dts.cmake

@@ -22,7 +22,8 @@ set(ZEPHYR_DTS                  ${PROJECT_BINARY_DIR}/zephyr.dts)
 set(EDT_PICKLE                  ${PROJECT_BINARY_DIR}/edt.pickle)
 set(DEVICETREE_UNFIXED_H        ${PROJECT_BINARY_DIR}/include/generated/devicetree_unfixed.h)
 set(DEVICE_EXTERN_H             ${PROJECT_BINARY_DIR}/include/generated/device_extern.h)
-set(DTS_POST_CPP                ${PROJECT_BINARY_DIR}/${BOARD}.dts.pre.tmp)
+set(DTS_POST_CPP                ${PROJECT_BINARY_DIR}/zephyr.dts.pre)
+set(DTS_DEPS                    ${PROJECT_BINARY_DIR}/zephyr.dts.d)
 # The location of a list of known vendor prefixes.
 # This is relative to each element of DTS_ROOT.
 set(VENDOR_PREFIXES             dts/bindings/vendor-prefixes.txt)
@@ -141,10 +142,9 @@ if(SUPPORTS_DTS)
   # preprocessor, and it seems to be including all kinds of
   # directories with who-knows how many header files.
 
-  # Run the C preprocessor on an empty C source file that has one or
-  # more DTS source files -include'd into it to create the
-  # intermediary file *.dts.pre.tmp. Also, generate a dependency file
-  # so that changes to DT sources are detected.
+  # Run the preprocessor on the DTS input files. We are leaving
+  # linemarker directives enabled on purpose. This tells dtlib where
+  # each line actually came from, which improves error reporting.
   execute_process(
     COMMAND ${CMAKE_DTS_PREPROCESSOR}
     -x assembler-with-cpp
@@ -154,11 +154,10 @@ if(SUPPORTS_DTS)
     ${NOSYSDEF_CFLAG}
     -D__DTS__
     ${DTS_EXTRA_CPPFLAGS}
-    -P
     -E   # Stop after preprocessing
     -MD  # Generate a dependency file as a side-effect
-    -MF ${PROJECT_BINARY_DIR}/${BOARD}.dts.pre.d
-    -o  ${PROJECT_BINARY_DIR}/${BOARD}.dts.pre.tmp
+    -MF ${DTS_DEPS}
+    -o ${DTS_POST_CPP}
     ${ZEPHYR_BASE}/misc/empty_file.c
     WORKING_DIRECTORY ${APPLICATION_SOURCE_DIR}
     RESULT_VARIABLE ret
@@ -168,10 +167,9 @@ if(SUPPORTS_DTS)
   endif()
 
   # Parse the generated dependency file to find the DT sources that
-  # were included and then add them to the list of files that trigger
-  # a re-run of CMake.
-  toolchain_parse_make_rule(
-    ${PROJECT_BINARY_DIR}/${BOARD}.dts.pre.d
+  # were included, including any transitive includes, and then add
+  # them to the list of files that trigger a re-run of CMake.
+  toolchain_parse_make_rule(${DTS_DEPS}
     include_files # Output parameter
     )
 
@@ -188,12 +186,12 @@ if(SUPPORTS_DTS)
 
   string(REPLACE ";" " " EXTRA_DTC_FLAGS_RAW "${EXTRA_DTC_FLAGS}")
   set(CMD_EXTRACT ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
-  --dts ${BOARD}.dts.pre.tmp
+  --dts ${DTS_POST_CPP}
   --dtc-flags '${EXTRA_DTC_FLAGS_RAW}'
   --bindings-dirs ${DTS_ROOT_BINDINGS}
   --header-out ${DEVICETREE_UNFIXED_H}
   --device-header-out ${DEVICE_EXTERN_H}
-  --dts-out ${ZEPHYR_DTS} # As a debugging aid
+  --dts-out ${ZEPHYR_DTS} # for debugging and dtc
   --edt-pickle-out ${EDT_PICKLE}
   ${EXTRA_GEN_DEFINES_ARGS}
   )
@@ -226,9 +224,9 @@ if(SUPPORTS_DTS)
   endif()
 
   #
-  # Run the C devicetree compiler on *.dts.pre.tmp, just to catch any
+  # Run dtc on the final devicetree source, just to catch any
   # warnings/errors from it. dtlib and edtlib parse the devicetree files
-  # themselves, so we don't rely on the C compiler otherwise.
+  # themselves, so we don't rely on dtc otherwise.
   #
 
   if(DTC)
@@ -259,7 +257,7 @@ if(SUPPORTS_DTS)
     ${DTC_NO_WARN_UNIT_ADDR}
     ${DTC_WARN_UNIT_ADDR_IF_ENABLED}
     ${EXTRA_DTC_FLAGS} # User settable
-    ${BOARD}.dts.pre.tmp
+    ${ZEPHYR_DTS}
     OUTPUT_QUIET # Discard stdout
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
     RESULT_VARIABLE ret