From 49206a86ff3988cfdc21a9ba4562217559b5e670 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Fri, 7 Aug 2020 10:47:37 -0700 Subject: [PATCH] debug/coredump: add a primitive coredump mechanism This adds a very primitive coredump mechanism under subsys/debug where during fatal error, register and memory content can be dumped to coredump backend. One such backend utilizing log module for output is included. Once the coredump log is converted to a binary file, it can be used with the ELF output file as inputs to an overly simplified implementation of a GDB server. This GDB server can be attached via the target remote command of GDB and will be serving register and memory content. This allows using GDB to examine stack and memory where the fatal error occurred. Signed-off-by: Daniel Leung --- CODEOWNERS | 3 + arch/Kconfig | 3 + doc/guides/debugging/coredump.rst | 347 ++++++++++++++++++ doc/guides/debugging/index.rst | 1 + include/debug/coredump.h | 141 +++++++ kernel/fatal.c | 3 + kernel/include/kernel_arch_interface.h | 22 ++ scripts/coredump/coredump_gdbserver.py | 132 +++++++ .../coredump/coredump_serial_log_parser.py | 99 +++++ scripts/coredump/gdbstubs/__init__.py | 15 + scripts/coredump/gdbstubs/arch/__init__.py | 5 + scripts/coredump/gdbstubs/gdbstub.py | 205 +++++++++++ scripts/coredump/parser/__init__.py | 5 + scripts/coredump/parser/elf_parser.py | 93 +++++ scripts/coredump/parser/log_parser.py | 167 +++++++++ subsys/debug/CMakeLists.txt | 5 + subsys/debug/Kconfig | 2 + subsys/debug/coredump/CMakeLists.txt | 19 + subsys/debug/coredump/Kconfig | 50 +++ .../debug/coredump/coredump_backend_logging.c | 93 +++++ subsys/debug/coredump/coredump_core.c | 186 ++++++++++ subsys/debug/coredump/coredump_internal.h | 74 ++++ .../debug/coredump/coredump_memory_regions.c | 22 ++ 23 files changed, 1692 insertions(+) create mode 100644 doc/guides/debugging/coredump.rst create mode 100644 include/debug/coredump.h create mode 100755 scripts/coredump/coredump_gdbserver.py create mode 100755 scripts/coredump/coredump_serial_log_parser.py create mode 100644 scripts/coredump/gdbstubs/__init__.py create mode 100644 scripts/coredump/gdbstubs/arch/__init__.py create mode 100644 scripts/coredump/gdbstubs/gdbstub.py create mode 100644 scripts/coredump/parser/__init__.py create mode 100644 scripts/coredump/parser/elf_parser.py create mode 100644 scripts/coredump/parser/log_parser.py create mode 100644 subsys/debug/coredump/CMakeLists.txt create mode 100644 subsys/debug/coredump/Kconfig create mode 100644 subsys/debug/coredump/coredump_backend_logging.c create mode 100644 subsys/debug/coredump/coredump_core.c create mode 100644 subsys/debug/coredump/coredump_internal.h create mode 100644 subsys/debug/coredump/coredump_memory_regions.c diff --git a/CODEOWNERS b/CODEOWNERS index 8bbd7df9c20..ae74dc02b27 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -354,6 +354,7 @@ /include/canbus/ @alexanderwachter /include/tracing/ @wentongwu @nashif /include/debug/ @nashif +/include/debug/coredump.h @dcpleung /include/device.h @wentongwu @nashif /include/devicetree.h @galak /include/display/ @vanwinkeljan @@ -426,6 +427,7 @@ /samples/userspace/ @andrewboie /scripts/coccicheck @himanshujha199640 @JuliaLawall /scripts/coccinelle/ @himanshujha199640 @JuliaLawall +/scripts/coredump/ @dcpleung /scripts/kconfig/ @ulfalizer /scripts/sanity_chk/expr_parser.py @nashif /scripts/gen_app_partitions.py @andrewboie @@ -460,6 +462,7 @@ /subsys/canbus/ @alexanderwachter /subsys/cpp/ @pabigot @vanwinkeljan /subsys/debug/ @nashif +/subsys/debug/coredump/ @dcpleung /subsys/dfu/ @nvlsianpu /subsys/tracing/ @nashif @wentongwu /subsys/debug/asan_hacks.c @vanwinkeljan @aescolar @daor-oti diff --git a/arch/Kconfig b/arch/Kconfig index 8f7a00370f4..8ce3b975366 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -392,6 +392,9 @@ config ARCH_HAS_RAMFUNC_SUPPORT config ARCH_HAS_NESTED_EXCEPTION_DETECTION bool +config ARCH_SUPPORTS_COREDUMP + bool + # # Other architecture related options # diff --git a/doc/guides/debugging/coredump.rst b/doc/guides/debugging/coredump.rst new file mode 100644 index 00000000000..a1c1c97550d --- /dev/null +++ b/doc/guides/debugging/coredump.rst @@ -0,0 +1,347 @@ +.. _coredump: + +Core Dump +######### + +The core dump module enables dumping the CPU registers and memory content +for offline debugging. This module is called when fatal error is +encountered, and the data is printed or stored according to which backends +are enabled. + +Configuration +************* + +Configure this module using the following options. + +* ``DEBUG_COREDUMP``: enable the module. + +Here are the options to enable output backends for core dump: + +* ``DEBUG_COREDUMP_BACKEND_LOGGING``: use log module for core dump output. +* ``DEBUG_COREDUMP_BACKEND_NULL``: fallback core dump backend if other + backends cannot be enabled. All output is sent to null. + +Here are the choices regarding memory dump: + +* ``DEBUG_COREDUMP_MEMORY_DUMP_MIN```: only dumps the stack of the exception + thread, its thread struct, and some other bare minimal data to support + walking the stack in debugger. Use this only if absolute minimum of data + dump is desired. + +Usage +***** + +When the core dump module is enabled, during fatal error, CPU registers +and memory content are being printed or stored according to which backends +are enabled. This core dump data can fed into a custom made GDB server as +a remote target for GDB (and other GDB compatible debuggers). CPU registers, +memory content and stack can be examined in the debugger. + +This usually involves the following steps: + +1. Get the core dump log from the device depending on enabled backends. + For example, if the log module backend is used, get the log output + from the log module backend. + +2. Convert the core dump log into a binary format that can be parsed by + the GDB server. For example, + :zephyr_file:`scripts/coredump/coredump_serial_log_parser.py` can be used + to convert the serial console log into a binary file. + +3. Start the custom GDB server using the script + :zephyr_file:`scripts/coredump/coredump_gdbserver.py` with the core dump + binary log file, and the Zephyr ELF file as parameters. + +4. Start the debugger corresponding to the target architecture. + +Example +------- + +This example uses the log module backend tied to serial console. +This was done on `qemu_x86` where a null pointer was dereferenced. + +This is the core dump log from the serial console, and is stored +in :file:`coredump.log`: + +:: + + Booting from ROM..*** Booting Zephyr OS build zephyr-v2.3.0-1840-g7bba91944a63 *** + Hello World! qemu_x86 + E: Page fault at address 0x0 (error code 0x2) + E: Linear address not present in page tables + E: PDE: 0x0000000000115827 Writable, User, Execute Enabled + E: PTE: Non-present + E: EAX: 0x00000000, EBX: 0x00000000, ECX: 0x00119d74, EDX: 0x000003f8 + E: ESI: 0x00000000, EDI: 0x00101aa7, EBP: 0x00119d10, ESP: 0x00119d00 + E: EFLAGS: 0x00000206 CS: 0x0008 CR3: 0x00119000 + E: call trace: + E: EIP: 0x00100459 + E: 0x00100477 (0x0) + E: 0x00100492 (0x0) + E: 0x001004c8 (0x0) + E: 0x00105465 (0x105465) + E: 0x00101abe (0x0) + E: >>> ZEPHYR FATAL ERROR 0: CPU exception on CPU 0 + E: Current thread: 0x00119080 (unknown) + E: #CD:BEGIN# + E: #CD:5a4501000100050000000000 + E: #CD:4101003800 + E: #CD:0e0000000200000000000000749d1100f803000000000000009d1100109d1100 + E: #CD:00000000a71a100059041000060200000800000000901100 + E: #CD:4d010080901100e0901100 + E: #CD:0100000000000000000000000180000000000000000000000000000000000000 + E: #CD:00000000000000000000000000000000e364100000000000000000004c9c1100 + E: #CD:000000000000000000000000b49911000004000000000000fc03000000000000 + E: #CD:4d0100b4991100b49d1100 + E: #CD:f8030000020000000200000002000000f8030000fd03000a02000000dc9e1100 + E: #CD:149a1160fd03000002000000dc9e1100249a110087201000049f11000a000000 + E: #CD:349a11000a4f1000049f11000a9e1100449a11000a8b10000200000002000000 + E: #CD:449a1100388b1000049f11000a000000549a1100ad201000049f11000a000000 + E: #CD:749a11000a201000049f11000a000000649a11000a201000049f11000a000000 + E: #CD:749a1100e8201000049f11000a000000949a1100890b10000a0000000a000000 + E: #CD:a49a1100890b10000a0000000a000000f8030000189b11000200000002000000 + E: #CD:f49a1100289b11000a000000189b1100049b11009b0710000a000000289b1100 + E: #CD:f49a110087201000049f110045000000f49a1100509011000a00000020901100 + E: #CD:f49a110060901100049f1100ffffffff0000000000000000049f1100ffffffff + E: #CD:0000000000000000630b1000189b1100349b1100af0b1000630b1000289b1100 + E: #CD:55891000789b11000000000020901100549b1100480000004a891000609b1100 + E: #CD:649b1100d00b10004a891000709b110000000000609b11000a00000000000000 + E: #CD:849b1100709b11004a89100000000000949b1100794a10000000000058901100 + E: #CD:20901100c34a10000a00001734020000d001000000000000d49b110038000000 + E: #CD:c49b110078481000b49911000004000000000000000000000c9c11000c9c1100 + E: #CD:149c110000000000d49b110038000000f49b1100da481000b499110000040000 + E: #CD:0e0000000200000000000000744d0100b4991100b49d1100009d1100109d1100 + E: #CD:149c110099471000b4991100000400000800000000901100ad861000409c1100 + E: #CD:349c1100e94710008090110000000000349c1100b64710008086100045000000 + E: #CD:849c11002d53100000000000d09c11008090110020861000f5ffffff8c9c1100 + E: #CD:000000000000000000000000a71a1000a49c1100020200008090110000000000 + E: #CD:a49c1100020200000800000000000000a49c11001937100000000000d09c1100 + E: #CD:0c9d0000bc9c0000b49d1100b4991100c49c1100ae37100000000000d09c1100 + E: #CD:0800000000000000c888100000000000109d11005d031000d09c1100009d1100 + E: #CD:109d11000000000000000000a71a1000f803000000000000749d110002000000 + E: #CD:5904100008000000060200000e0000000202000002020000000000002c9d1100 + E: #CD:7704100000000000d00b1000c9881000549d110000000000489d110092041000 + E: #CD:00000000689d1100549d11000000000000000000689d1100c804100000000000 + E: #CD:c0881000000000007c9d110000000000749d11007c9d11006554100065541000 + E: #CD:00000000000000009c9d1100be1a100000000000000000000000000038041000 + E: #CD:08000000020200000000000000000000f4531000000000000000000000000000 + E: #CD:END# + E: Halting system + + +1. Run the core dump serial log converter: + + .. code-block:: console + + ./scripts/coredump/coredump_serial_log_parser.py coredump.log coredump.bin + +2. Start the custom GDB server: + + .. code-block:: console + + ./scripts/coredump/coredump_gdbserver.py build/zephyr/zephyr.elf coredump.bin + +3. Start GDB: + + .. code-block:: console + + /x86_64-zephyr-elf/bin/x86_64-zephyr-elf-gdb build/zephyr/zephyr.elf + +4. Inside GDB, connect to the GDB server via port 1234: + + .. code-block:: console + + (gdb) target remote localhost:1234 + +5. Examine the CPU registers: + + .. code-block:: console + + (gdb) info registers + + Output from GDB: + + :: + + eax 0x0 0 + ecx 0x119d74 1154420 + edx 0x3f8 1016 + ebx 0x0 0 + esp 0x119d00 0x119d00 + ebp 0x119d10 0x119d10 + esi 0x0 0 + edi 0x101aa7 1055399 + eip 0x100459 0x100459 + eflags 0x206 [ PF IF ] + cs 0x8 8 + ss + ds + es + fs + gs + +6. Examine the backtrace: + + .. code-block:: console + + (gdb) bt + + + Output from GDB: + + :: + + #0 0x00100459 in func_3 (addr=0x0) at zephyr/rtos/zephyr/samples/hello_world/src/main.c:14 + #1 0x00100477 in func_2 (addr=0x0) at zephyr/rtos/zephyr/samples/hello_world/src/main.c:21 + #2 0x00100492 in func_1 (addr=0x0) at zephyr/rtos/zephyr/samples/hello_world/src/main.c:28 + #3 0x001004c8 in main () at zephyr/rtos/zephyr/samples/hello_world/src/main.c:42 + +File Format +*********** + +The core dump binary file consists of one file header, one +architecture-specific block, and multiple memory blocks. All numbers in +the headers below are little endian. + +File Header +----------- + +The file header consists of the following fields: + +.. list-table:: Core dump binary file header + :widths: 2 1 7 + :header-rows: 1 + + * - Field + - Data Type + - Description + * - ID + - ``char[2]`` + - ``Z``, ``E`` as identifier of file. + * - Header version + - ``uint16_t`` + - Identify the version of the header. This needs to be incremented + whenever the header struct is modified. This allows parser to + reject older header versions so it will not incorrectly parse + the header. + * - Target code + - ``uint16_t`` + - Indicate which target (e.g. architecture or SoC) so the parser + can instantiate the correct register block parser. + * - Pointer size + - 'uint8_t' + - Size of ``uintptr_t`` in power of 2. (e.g. 5 for 32-bit, + 6 for 64-bit). This is needed to accommodate 32-bit and 64-bit + target in parsing the memory block addresses. + * - Flags + - ``uint8_t`` + - + * - Fatal error reason + - ``unsigned int`` + - Reason for the fatal error, as the same in + ``enum k_fatal_error_reason`` defined in + :zephyr_file:`include/fatal.h` + +Architecture-specific Block +--------------------------- + +The architecture-specific block contains the byte stream of data specific +to the target architecture (e.g. CPU registers) + +.. list-table:: Architecture-specific Block + :widths: 2 1 7 + :header-rows: 1 + + * - Field + - Data Type + - Description + * - ID + - ``char`` + - ``A`` to indiciate this is a architecture-specific block. + * - Header version + - ``uint16_t`` + - Identify the version of this block. To be interpreted by the target + architecture specific block parser. + * - Number of bytes + - ``uint16_t`` + - Number of bytes following the header which contains the byte stream + for target data. The format of the byte stream is specific to + the target and is only being parsed by the target parser. + * - Register byte stream + - ``uint8_t[]`` + - Contains target architecture specific data. + +Memory Block +------------ + +The memory block contains the start and end addresses and the data within +the memory region. + +.. list-table:: Memory Block + :widths: 2 1 7 + :header-rows: 1 + + * - Field + - Data Type + - Description + * - ID + - ``char`` + - ``M`` to indiciate this is a memory block. + * - Header version + - ``uint16_t`` + - Identify the version of the header. This needs to be incremented + whenever the header struct is modified. This allows parser to + reject older header versions so it will not incorrectly parse + the header. + * - Start address + - ``uintptr_t`` + - The start address of the memory region. + * - End address + - ``uintptr_t`` + - The end address of the memory region. + * - Memory byte stream + - ``uint8_t[]`` + - Contains the memory content between the start and end addresses. + +Adding New Target +***************** + +The architecture-specific block is target specific and requires new +dumping routine and parser for new targets. To add a new target, +the following needs to be done: + +#. Add a new target code to the ``enum z_coredump_tgt_code`` in + :zephyr_file:`include/debug/coredump.h`. +#. Implement :c:func:`arch_coredump_tgt_code_get` simply to return + the newly introducted target code. +#. Implement :c:func:`arch_coredump_info_dump` to construct + a target architecture block and call :c:func:`z_coredump_buffer_output` + to output the block to core dump backend. +#. Add a parser to the core dump GDB stub scripts under + ``scripts/coredump/gdbstubs/`` + + #. Extends the ``gdbstubs.gdbstub.GdbStub`` class. + #. During ``__init__``, store the GDB signal corresponding to + the exception reason in ``self.gdb_signal``. + #. Parse the architecture-specific block from + ``self.logfile.get_arch_data()``. This needs to match the format + as implemented in step 3 (inside :c:func:`arch_coredump_info_dump`). + #. Implement the abstract method ``handle_register_group_read_packet`` + where it returns the register group as GDB expected. Refer to + GDB's code and documentation on what it is expecting for + the new target. + #. Optionally implement ``handle_register_single_read_packet`` + for registers not covered in the ``g`` packet. + +#. Extend ``get_gdbstub()`` in + :zephyr_file:`scripts/coredump/gdbstubs/__init__.py` to return + the newly implemented GDB stub. + +API documentation +***************** + +.. doxygengroup:: coredump_apis + :project: Zephyr + +.. doxygengroup:: arch-coredump + :project: Zephyr diff --git a/doc/guides/debugging/index.rst b/doc/guides/debugging/index.rst index fae592c51ae..5a9d86ee75b 100644 --- a/doc/guides/debugging/index.rst +++ b/doc/guides/debugging/index.rst @@ -9,3 +9,4 @@ Debugging host-tools.rst probes.rst thread-analyzer.rst + coredump.rst diff --git a/include/debug/coredump.h b/include/debug/coredump.h new file mode 100644 index 00000000000..ada94ce4228 --- /dev/null +++ b/include/debug/coredump.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2020 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_DEBUG_COREDUMP_H_ +#define ZEPHYR_INCLUDE_DEBUG_COREDUMP_H_ + +#ifdef CONFIG_DEBUG_COREDUMP + +#include +#include +#include + +#define Z_COREDUMP_HDR_VER 1 + +#define Z_COREDUMP_ARCH_HDR_ID 'A' + +#define Z_COREDUMP_MEM_HDR_ID 'M' +#define Z_COREDUMP_MEM_HDR_VER 1 + +/* Target code */ +enum z_coredump_tgt_code { + COREDUMP_TGT_UNKNOWN = 0, +}; + +/* Coredump header */ +struct z_coredump_hdr_t { + /* 'Z', 'E' */ + char id[2]; + + /* Header version */ + uint16_t hdr_version; + + /* Target code */ + uint16_t tgt_code; + + /* Pointer size in Log2 */ + uint8_t ptr_size_bits; + + uint8_t flag; + + /* Coredump Reason given */ + unsigned int reason; +} __packed; + +/* Architecture-specific block header */ +struct z_coredump_arch_hdr_t { + /* Z_COREDUMP_ARCH_HDR_ID */ + char id; + + /* Header version */ + uint16_t hdr_version; + + /* Number of bytes in this block (excluding header) */ + uint16_t num_bytes; +} __packed; + +/* Memory block header */ +struct z_coredump_mem_hdr_t { + /* Z_COREDUMP_MEM_HDR_ID */ + char id; + + /* Header version */ + uint16_t hdr_version; + + /* Address of start of memory region */ + uintptr_t start; + + /* Address of end of memory region */ + uintptr_t end; +} __packed; + +void z_coredump(unsigned int reason, const z_arch_esf_t *esf, + struct k_thread *thread); +void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr); +int z_coredump_buffer_output(uint8_t *buf, size_t buflen); + +#else + +void z_coredump(unsigned int reason, const z_arch_esf_t *esf, + struct k_thread *thread) +{ +} + +void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr) +{ +} + +int z_coredump_buffer_output(uint8_t *buf, size_t buflen) +{ + return 0; +} + +#endif /* CONFIG_DEBUG_COREDUMP */ + +/** + * @defgroup coredump_apis Coredump APIs + * @brief Coredump APIs + * @{ + */ + +/** + * @fn void z_coredump(unsigned int reason, const z_arch_esf_t *esf, struct k_thread *thread); + * @brief Perform coredump. + * + * Normally, this is called inside z_fatal_error() to generate coredump + * when a fatal error is encountered. This can also be called on demand + * whenever a coredump is desired. + * + * @param reason Reason for the fatal error + * @param esf Exception context + * @param thread Thread information to dump + */ + +/** + * @fn void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr); + * @brief Dump memory region + * + * @param start_addr Start address of memory region to be dumped + * @param emd_addr End address of memory region to be dumped + */ + +/** + * @fn int z_coredump_buffer_output(uint8_t *buf, size_t buflen); + * @brief Output the buffer via coredump + * + * This outputs the buffer of byte array to the coredump backend. + * For example, this can be called to output the coredump section + * containing registers, or a section for memory dump. + * + * @param buf Buffer to be send to coredump output + * @param buflen Buffer length + */ + +/** + * @} + */ + +#endif /* ZEPHYR_INCLUDE_DEBUG_COREDUMP_H_ */ diff --git a/kernel/fatal.c b/kernel/fatal.c index f3a149c6b55..1922e320aca 100644 --- a/kernel/fatal.c +++ b/kernel/fatal.c @@ -13,6 +13,7 @@ #include #include #include +#include LOG_MODULE_DECLARE(os); @@ -120,6 +121,8 @@ void z_fatal_error(unsigned int reason, const z_arch_esf_t *esf) LOG_ERR("Current thread: %p (%s)", thread, log_strdup(thread_name_get(thread))); + z_coredump(reason, esf, thread); + k_sys_fatal_error_handler(reason, esf); /* If the system fatal error handler returns, then kill the faulting diff --git a/kernel/include/kernel_arch_interface.h b/kernel/include/kernel_arch_interface.h index 222ae223d1c..c36e9413807 100644 --- a/kernel/include/kernel_arch_interface.h +++ b/kernel/include/kernel_arch_interface.h @@ -299,6 +299,28 @@ static inline void arch_nop(void); /** @} */ +/** + * @defgroup arch-coredump Architecture-specific core dump APIs + * @ingroup arch-interface + * @{ + */ + +/** + * @brief Architecture-specific handling during coredump + * + * This dumps architecture-specific information during coredump. + * + * @param esf Exception Stack Frame (arch-specific) + */ +void arch_coredump_info_dump(const z_arch_esf_t *esf); + +/** + * @brief Get the target code specified by the architecture. + */ +uint16_t arch_coredump_tgt_code_get(void); + +/** @} */ + /* Include arch-specific inline function implementation */ #include diff --git a/scripts/coredump/coredump_gdbserver.py b/scripts/coredump/coredump_gdbserver.py new file mode 100755 index 00000000000..8391b435bf8 --- /dev/null +++ b/scripts/coredump/coredump_gdbserver.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import logging +import os +import socket +import sys + +from parser.log_parser import CoredumpLogFile +from parser.elf_parser import CoredumpElfFile + +import gdbstubs + +LOGGING_FORMAT = "[%(levelname)s][%(name)s] %(message)s" + +# Only bind to local host +GDBSERVER_HOST = "" + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("elffile", help="Zephyr ELF binary") + parser.add_argument("logfile", help="Coredump binary log file") + parser.add_argument("--debug", action="store_true", + help="Print extra debugging information") + parser.add_argument("--port", type=int, default=1234, + help="GDB server port") + parser.add_argument("-v", "--verbose", action="store_true", + help="Print more information") + + return parser.parse_args() + + +def main(): + args = parse_args() + + # Setup logging + logging.basicConfig(format=LOGGING_FORMAT) + + # Setup logging for "parser" + logger = logging.getLogger("parser") + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.verbose: + logger.setLevel(logging.INFO) + else: + logger.setLevel(logging.WARNING) + + # Setup logging for follow code + logger = logging.getLogger("gdbserver") + if args.debug: + logger.setLevel(logging.DEBUG) + else: + # Use INFO as default since we need to let user + # know what is going on + logger.setLevel(logging.INFO) + + # Setup logging for "gdbstuc" + logger = logging.getLogger("gdbstub") + if args.debug: + logger.setLevel(logging.DEBUG) + elif args.verbose: + logger.setLevel(logging.INFO) + else: + logger.setLevel(logging.WARNING) + + if not os.path.isfile(args.elffile): + logger.error(f"Cannot find file {args.elffile}, exiting...") + sys.exit(1) + + if not os.path.isfile(args.logfile): + logger.error(f"Cannot find file {args.logfile}, exiting...") + sys.exit(1) + + logger.info(f"Log file: {args.logfile}") + logger.info(f"ELF file: {args.elffile}") + + # Parse the coredump binary log file + logf = CoredumpLogFile(args.logfile) + logf.open() + if not logf.parse(): + logger.error("Cannot parse log file, exiting...") + logf.close() + sys.exit(1) + + # Parse ELF file for code and read-only data + elff = CoredumpElfFile(args.elffile) + elff.open() + if not elff.parse(): + logger.error("Cannot parse ELF file, exiting...") + elff.close() + logf.close() + sys.exit(1) + + gdbstub = gdbstubs.get_gdbstub(logf, elff) + + # Start a GDB server + gdbserver = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Reuse address so we don't have to wait for socket to be + # close before we can bind to the port again + gdbserver.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + gdbserver.bind((GDBSERVER_HOST, args.port)) + gdbserver.listen(1) + + logger.info(f"Waiting GDB connection on port {args.port}...") + + conn, remote = gdbserver.accept() + + if conn: + logger.info(f"Accepted GDB connection from {remote}") + + gdbstub.run(conn) + + conn.close() + + gdbserver.close() + + logger.info("GDB session finished.") + + elff.close() + logf.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/coredump/coredump_serial_log_parser.py b/scripts/coredump/coredump_serial_log_parser.py new file mode 100755 index 00000000000..32a7f4d7374 --- /dev/null +++ b/scripts/coredump/coredump_serial_log_parser.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import binascii +import sys + + +COREDUMP_PREFIX_STR = "#CD:" + +COREDUMP_BEGIN_STR = COREDUMP_PREFIX_STR + "BEGIN#" +COREDUMP_END_STR = COREDUMP_PREFIX_STR + "END#" +COREDUMP_ERROR_STR = COREDUMP_PREFIX_STR + "ERROR CANNOT DUMP#" + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("infile", help="Serial Log File") + parser.add_argument("outfile", + help="Output file for use with coredump GDB server") + + return parser.parse_args() + + +def main(): + args = parse_args() + + infile = open(args.infile, "r") + if not infile: + print(f"ERROR: Cannot open input file: {args.infile}, exiting...") + sys.exit(1) + + outfile = open(args.outfile, "wb") + if not outfile: + print(f"ERROR: Cannot open output file for write: {args.outfile}, exiting...") + sys.exit(1) + + print(f"Input file {args.infile}") + print(f"Output file {args.outfile}") + + has_begin = False + has_end = False + has_error = False + go_parse_line = False + bytes_written = 0 + for line in infile.readlines(): + if line.find(COREDUMP_BEGIN_STR) >= 0: + # Found "BEGIN#" - beginning of log + has_begin = True + go_parse_line = True + continue + + if line.find(COREDUMP_END_STR) >= 0: + # Found "END#" - end of log + has_end = True + go_parse_line = False + break + + if line.find(COREDUMP_ERROR_STR) >= 0: + # Error was encountered during dumping: + # log is not usable + has_error = True + go_parse_line = False + break + + if not go_parse_line: + continue + + prefix_idx = line.find(COREDUMP_PREFIX_STR) + + if prefix_idx < 0: + continue + + prefix_idx += len(COREDUMP_PREFIX_STR) + hex_str = line[prefix_idx:].strip() + + binary_data = binascii.unhexlify(hex_str) + outfile.write(binary_data) + bytes_written += len(binary_data) + + if not has_begin: + print("ERROR: Beginning of log not found!") + elif not has_end: + print("WARN: End of log not found! Is log complete?") + elif has_error: + print("ERROR: log has error.") + else: + print(f"Bytes written {bytes_written}") + + infile.close() + outfile.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/coredump/gdbstubs/__init__.py b/scripts/coredump/gdbstubs/__init__.py new file mode 100644 index 00000000000..76c04d0bfdd --- /dev/null +++ b/scripts/coredump/gdbstubs/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +class TgtCode: + UNKNOWN = 0 + +def get_gdbstub(logfile, elffile): + stub = None + + tgt_code = logfile.log_hdr['tgt_code'] + + return stub diff --git a/scripts/coredump/gdbstubs/arch/__init__.py b/scripts/coredump/gdbstubs/arch/__init__.py new file mode 100644 index 00000000000..7a792bb2556 --- /dev/null +++ b/scripts/coredump/gdbstubs/arch/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/coredump/gdbstubs/gdbstub.py b/scripts/coredump/gdbstubs/gdbstub.py new file mode 100644 index 00000000000..23b9d57c875 --- /dev/null +++ b/scripts/coredump/gdbstubs/gdbstub.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import abc +import binascii +import logging + + +logger = logging.getLogger("gdbstub") + + +class GdbStub(abc.ABC): + def __init__(self, logfile, elffile): + self.logfile = logfile + self.elffile = elffile + self.socket = None + self.gdb_signal = None + + mem_regions = list() + + for r in logfile.get_memory_regions(): + mem_regions.append(r) + + for r in elffile.get_memory_regions(): + mem_regions.append(r) + + self.mem_regions = mem_regions + + def get_gdb_packet(self): + socket = self.socket + if socket is None: + return None + + data = b'' + checksum = 0 + # Wait for '$' + while True: + ch = socket.recv(1) + if ch == b'$': + break + + # Get a full packet + while True: + ch = socket.recv(1) + if ch == b'#': + # End of packet + break + + checksum += ord(ch) + data += ch + + # Get checksum (2-bytes) + ch = socket.recv(2) + in_chksum = ord(binascii.unhexlify(ch)) + + logger.debug(f"Received GDB packet: {data}") + + if (checksum % 256) == in_chksum: + # ACK + logger.debug("ACK") + socket.send(b'+') + + return data + else: + # NACK + logger.debug(f"NACK (checksum {in_chksum} != {checksum}") + socket.send(b'-') + + return None + + def put_gdb_packet(self, data): + socket = self.socket + if socket is None: + return + + checksum = 0 + for d in data: + checksum += d + + pkt = b'$' + data + b'#' + + checksum = checksum % 256 + pkt += format(checksum, "02X").encode() + + logger.debug(f"Sending GDB packet: {pkt}") + + socket.send(pkt) + + def handle_signal_query_packet(self): + # the '?' packet + pkt = b'S' + pkt += format(self.gdb_signal, "02X").encode() + + self.put_gdb_packet(pkt) + + @abc.abstractmethod + def handle_register_group_read_packet(self): + # the 'g' packet for reading a group of registers + pass + + def handle_register_group_write_packet(self): + # the 'G' packet for writing to a group of registers + # + # We don't support writing so return error + self.put_gdb_packet(b"E01") + + def handle_register_single_read_packet(self, pkt): + # the 'p' packet for reading a single register + self.put_gdb_packet(b"E01") + + def handle_register_single_write_packet(self, pkt): + # the 'P' packet for writing to registers + # + # We don't support writing so return error + self.put_gdb_packet(b"E01") + + def handle_memory_read_packet(self, pkt): + # the 'm' packet for reading memory: m, + + def get_mem_region(addr): + for r in self.mem_regions: + if r['start'] <= addr <= r['end']: + return r + + return None + + # extract address and length from packet + # and convert them into usable integer values + str_addr, str_length = pkt[1:].split(b',') + s_addr = int(b'0x' + str_addr, 16) + length = int(b'0x' + str_length, 16) + + # FIXME: Need more efficient way of extracting memory content + remaining = length + addr = s_addr + barray = b'' + r = get_mem_region(addr) + while remaining > 0: + if addr > r['end']: + r = get_mem_region(addr) + + if r is None: + barray = None + break + + offset = addr - r['start'] + barray += r['data'][offset:offset+1] + + addr += 1 + remaining -= 1 + + if barray is not None: + pkt = binascii.hexlify(barray) + self.put_gdb_packet(pkt) + else: + self.put_gdb_packet(b"E01") + + def handle_memory_write_packet(self, pkt): + # the 'M' packet for writing to memory + # + # We don't support writing so return error + self.put_gdb_packet(b"E02") + + def handle_general_query_packet(self, pkt): + self.put_gdb_packet(b'') + + def run(self, socket): + self.socket = socket + + while True: + pkt = self.get_gdb_packet() + if pkt is None: + continue + + pkt_type = pkt[0:1] + logger.debug(f"Got packet type: {pkt_type}") + + if pkt_type == b'?': + self.handle_signal_query_packet() + elif pkt_type in (b'C', b'S'): + # Continue/stepping execution, which is not supported. + # So signal exception again + self.handle_signal_query_packet() + elif pkt_type == b'g': + self.handle_register_group_read_packet() + elif pkt_type == b'G': + self.handle_register_group_write_packet() + elif pkt_type == b'p': + self.handle_register_single_read_packet(pkt) + elif pkt_type == b'P': + self.handle_register_single_write_packet(pkt) + elif pkt_type == b'm': + self.handle_memory_read_packet(pkt) + elif pkt_type == b'M': + self.handle_memory_write_packet(pkt) + elif pkt_type == b'q': + self.handle_general_query_packet(pkt) + elif pkt_type == b'k': + # GDB quits + break + else: + self.put_gdb_packet(b'') diff --git a/scripts/coredump/parser/__init__.py b/scripts/coredump/parser/__init__.py new file mode 100644 index 00000000000..7a792bb2556 --- /dev/null +++ b/scripts/coredump/parser/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/coredump/parser/elf_parser.py b/scripts/coredump/parser/elf_parser.py new file mode 100644 index 00000000000..013b6185bbd --- /dev/null +++ b/scripts/coredump/parser/elf_parser.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import logging + +import elftools +from elftools.elf.elffile import ELFFile + + +# ELF section flags +SHF_WRITE = 0x1 +SHF_ALLOC = 0x2 +SHF_EXEC = 0x4 +SHF_WRITE_ALLOC = SHF_WRITE | SHF_ALLOC +SHF_ALLOC_EXEC = SHF_ALLOC | SHF_EXEC + + +logger = logging.getLogger("parser") + + +class CoredumpElfFile(): + """ + Class to parse ELF file for memory content in various sections. + There are read-only sections (e.g. text and rodata) where + the memory content does not need to be dumped via coredump + and can be retrived from the ELF file. + """ + + def __init__(self, elffile): + self.elffile = elffile + self.fd = None + self.elf = None + self.memory_regions = list() + + def open(self): + self.fd = open(self.elffile, "rb") + self.elf = ELFFile(self.fd) + + def close(self): + self.fd.close() + + def get_memory_regions(self): + return self.memory_regions + + def parse(self): + if self.fd is None: + self.open() + + for section in self.elf.iter_sections(): + # REALLY NEED to match exact type as all other sections + # (debug, text, etc.) are descendants where + # isinstance() would match. + if type(section) is not elftools.elf.sections.Section: # pylint: disable=unidiomatic-typecheck + continue + + size = section['sh_size'] + flags = section['sh_flags'] + sec_start = section['sh_addr'] + sec_end = sec_start + size - 1 + + store = False + sect_desc = "?" + + if section['sh_type'] == 'SHT_PROGBITS': + if (flags & SHF_ALLOC_EXEC) == SHF_ALLOC_EXEC: + # Text section + store = True + sect_desc = "text" + elif (flags & SHF_WRITE_ALLOC) == SHF_WRITE_ALLOC: + # Data section + # + # Running app changes the content so no need + # to store + pass + elif (flags & SHF_ALLOC) == SHF_ALLOC: + # Read only data section + store = True + sect_desc = "read-only data" + + if store: + mem_region = {"start": sec_start, "end": sec_end, "data": section.data()} + logger.info("ELF Section: 0x%x to 0x%x of size %d (%s)" % + (mem_region["start"], + mem_region["end"], + len(mem_region["data"]), + sect_desc)) + + self.memory_regions.append(mem_region) + + return True diff --git a/scripts/coredump/parser/log_parser.py b/scripts/coredump/parser/log_parser.py new file mode 100644 index 00000000000..d6568e1d0e1 --- /dev/null +++ b/scripts/coredump/parser/log_parser.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import struct + + +# Note: keep sync with C code +Z_COREDUMP_HDR_ID = b'ZE' +Z_COREDUMP_HDR_VER = 1 +LOG_HDR_STRUCT = " +#include + +#include +#include "coredump_internal.h" + +#include +LOG_MODULE_DECLARE(coredump, CONFIG_KERNEL_LOG_LEVEL); + +#define COREDUMP_BEGIN_STR "BEGIN#" +#define COREDUMP_END_STR "END#" +#define COREDUMP_ERROR_STR "ERROR CANNOT DUMP#" + +/* + * Need to prefix coredump strings to make it easier to parse + * as log module adds its own prefixes. + */ +#define COREDUMP_PREFIX_STR "#CD:" + +/* Length of buffer of printable size */ +#define LOG_BUF_SZ 64 + +/* Length of buffer of printable size plus null character */ +#define LOG_BUF_SZ_RAW (LOG_BUF_SZ + 1) + +/* Log Buffer */ +char log_buf[LOG_BUF_SZ_RAW]; + +static void coredump_logging_backend_start(void) +{ + LOG_ERR(COREDUMP_PREFIX_STR COREDUMP_BEGIN_STR); +} + +static void coredump_logging_backend_end(void) +{ + LOG_ERR(COREDUMP_PREFIX_STR COREDUMP_END_STR); +} + +static void coredump_logging_backend_error(void) +{ + LOG_ERR(COREDUMP_PREFIX_STR COREDUMP_ERROR_STR); +} + +static int coredump_logging_backend_buffer_output(uint8_t *buf, size_t buflen) +{ + uint8_t log_ptr = 0; + size_t remaining = buflen; + size_t i = 0; + int ret = 0; + + if ((buf == NULL) || (buflen == 0)) { + ret = -EINVAL; + remaining = 0; + } + + while (remaining > 0) { + if (hex2char(buf[i] >> 4, &log_buf[log_ptr]) < 0) { + ret = -EINVAL; + break; + } + log_ptr++; + + if (hex2char(buf[i] & 0xf, &log_buf[log_ptr]) < 0) { + ret = -EINVAL; + break; + } + log_ptr++; + + i++; + remaining--; + + if ((log_ptr >= LOG_BUF_SZ) || (remaining == 0)) { + log_buf[log_ptr] = '\0'; + LOG_ERR(COREDUMP_PREFIX_STR "%s", log_buf); + log_ptr = 0; + } + } + + return ret; +} + +struct z_coredump_backend_api z_coredump_backend_logging = { + .start = coredump_logging_backend_start, + .end = coredump_logging_backend_end, + .error = coredump_logging_backend_error, + .buffer_output = coredump_logging_backend_buffer_output, +}; diff --git a/subsys/debug/coredump/coredump_core.c b/subsys/debug/coredump/coredump_core.c new file mode 100644 index 00000000000..28fe93a1e82 --- /dev/null +++ b/subsys/debug/coredump/coredump_core.c @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2020 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "coredump_internal.h" + +#if defined(CONFIG_DEBUG_COREDUMP_BACKEND_LOGGING) +extern struct z_coredump_backend_api z_coredump_backend_logging; +static struct z_coredump_backend_api + *backend_api = &z_coredump_backend_logging; +#elif defined(DEBUG_COREDUMP_BACKEND_NULL) +extern struct z_coredump_backend_api z_coredump_backend_null; +static struct z_coredump_backend_api + *backend_api = &z_coredump_backend_null; +#else +#error "Need to select a coredump backend" +#endif + +static int error; + +static int dump_header(unsigned int reason) +{ + struct z_coredump_hdr_t hdr = { + .id = {'Z', 'E'}, + .hdr_version = Z_COREDUMP_HDR_VER, + .reason = sys_cpu_to_le16(reason), + }; + + if (sizeof(uintptr_t) == 8) { + hdr.ptr_size_bits = 6; /* 2^6 = 64 */ + } else if (sizeof(uintptr_t) == 4) { + hdr.ptr_size_bits = 5; /* 2^5 = 32 */ + } else { + hdr.ptr_size_bits = 0; /* Unknown */ + } + + hdr.tgt_code = sys_cpu_to_le16(arch_coredump_tgt_code_get()); + + return backend_api->buffer_output((uint8_t *)&hdr, sizeof(hdr)); +} + +static void dump_thread(struct k_thread *thread) +{ +#ifdef CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN + uintptr_t end_addr; + + /* + * When dumping minimum information, + * the current thread struct and stack need to + * to be dumped so debugger can examine them. + */ + + if (thread == NULL) { + return; + } + + end_addr = POINTER_TO_UINT(thread) + sizeof(*thread); + + z_coredump_memory_dump(POINTER_TO_UINT(thread), end_addr); + + end_addr = thread->stack_info.start + thread->stack_info.size; + + z_coredump_memory_dump(thread->stack_info.start, end_addr); +#endif +} + +void process_memory_region_list(void) +{ +#ifdef CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_LINKER_RAM + unsigned int idx = 0; + + while (true) { + struct z_coredump_memory_region_t *r = + &z_coredump_memory_regions[idx]; + + if (r->end == POINTER_TO_UINT(NULL)) { + break; + } + + z_coredump_memory_dump(r->start, r->end); + + idx++; + } +#endif +} + +void z_coredump(unsigned int reason, const z_arch_esf_t *esf, + struct k_thread *thread) +{ + error = 0; + + z_coredump_start(); + + dump_header(reason); + + if (esf != NULL) { + arch_coredump_info_dump(esf); + } + + if (thread != NULL) { + dump_thread(thread); + } + + process_memory_region_list(); + + if (error != 0) { + z_coredump_error(); + } + + z_coredump_end(); +} + +void z_coredump_start(void) +{ + backend_api->start(); +} + +void z_coredump_end(void) +{ + backend_api->end(); +} + +void z_coredump_error(void) +{ + backend_api->error(); +} + +int z_coredump_buffer_output(uint8_t *buf, size_t buflen) +{ + int ret; + + /* Error encountered before, skip */ + if (error != 0) { + return -EAGAIN; + } + + if ((buf == NULL) || (buflen == 0)) { + ret = -EINVAL; + } else { + error = backend_api->buffer_output(buf, buflen); + ret = error; + } + + return ret; +} + +void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr) +{ + struct z_coredump_mem_hdr_t m; + size_t len; + + if ((start_addr == POINTER_TO_UINT(NULL)) || + (end_addr == POINTER_TO_UINT(NULL))) { + return; + } + + if (start_addr >= end_addr) { + return; + } + + len = end_addr - start_addr; + + m.id = Z_COREDUMP_MEM_HDR_ID; + m.hdr_version = Z_COREDUMP_MEM_HDR_VER; + + if (sizeof(uintptr_t) == 8) { + m.start = sys_cpu_to_le64(start_addr); + m.end = sys_cpu_to_le64(end_addr); + } else if (sizeof(uintptr_t) == 4) { + m.start = sys_cpu_to_le32(start_addr); + m.end = sys_cpu_to_le32(end_addr); + } + + z_coredump_buffer_output((uint8_t *)&m, sizeof(m)); + + z_coredump_buffer_output((uint8_t *)start_addr, len); +} diff --git a/subsys/debug/coredump/coredump_internal.h b/subsys/debug/coredump/coredump_internal.h new file mode 100644 index 00000000000..1611fa76c1f --- /dev/null +++ b/subsys/debug/coredump/coredump_internal.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef DEBUG_COREDUMP_INTERNAL_H_ +#define DEBUG_COREDUMP_INTERNAL_H_ + +#include + +/** + * @cond INTERNAL_HIDDEN + * + * These are for internal use only, so skip these in + * public documentation. + */ + +struct z_coredump_memory_region_t { + uintptr_t start; + uintptr_t end; +}; + +extern struct z_coredump_memory_region_t z_coredump_memory_regions[]; + +/** + * @brief Mark the start of coredump + * + * This sets up coredump subsys so coredump can be commenced. + * + * For example, backend needs to be initialized before any + * output can be stored. + */ +void z_coredump_start(void); + +/** + * @brief Mark the end of coredump + * + * This tells the coredump subsys to finalize the coredump + * session. + * + * For example, backend may need to flush the output. + */ +void z_coredump_end(void); + +/** + * @brief Signal to coredump subsys there is an error. + */ +void z_coredump_error(void); + +typedef void (*z_coredump_backend_start_t)(void); +typedef void (*z_coredump_backend_end_t)(void); +typedef void (*z_coredump_backend_error_t)(void); +typedef int (*z_coredump_backend_buffer_output_t)(uint8_t *buf, size_t buflen); + +struct z_coredump_backend_api { + /* Signal to backend of the start of coredump. */ + z_coredump_backend_start_t start; + + /* Signal to backend of the end of coredump. */ + z_coredump_backend_end_t end; + + /* Signal to backend an error has been encountered. */ + z_coredump_backend_error_t error; + + /* Raw buffer output */ + z_coredump_backend_buffer_output_t buffer_output; +}; + +/** + * @endcond + */ + +#endif /* DEBUG_COREDUMP_INTERNAL_H_ */ diff --git a/subsys/debug/coredump/coredump_memory_regions.c b/subsys/debug/coredump/coredump_memory_regions.c new file mode 100644 index 00000000000..25278757f04 --- /dev/null +++ b/subsys/debug/coredump/coredump_memory_regions.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2020 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "coredump_internal.h" + +#ifdef CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_LINKER_RAM +struct z_coredump_memory_region_t __weak z_coredump_memory_regions[] = { + {(uintptr_t)&_image_ram_start, (uintptr_t)&_image_ram_end}, + {0, 0} /* End of list */ +}; +#endif