debug/coredump: add a primitive coredump mechanism

This adds a very primitive coredump mechanism under subsys/debug
where during fatal error, register and memory content can be
dumped to coredump backend. One such backend utilizing log
module for output is included. Once the coredump log is converted
to a binary file, it can be used with the ELF output file as
inputs to an overly simplified implementation of a GDB server.
This GDB server can be attached via the target remote command of
GDB and will be serving register and memory content. This allows
using GDB to examine stack and memory where the fatal error
occurred.

Signed-off-by: Daniel Leung <daniel.leung@intel.com>
This commit is contained in:
Daniel Leung 2020-08-07 10:47:37 -07:00 committed by Anas Nashif
commit 49206a86ff
23 changed files with 1692 additions and 0 deletions

View file

@ -354,6 +354,7 @@
/include/canbus/ @alexanderwachter
/include/tracing/ @wentongwu @nashif
/include/debug/ @nashif
/include/debug/coredump.h @dcpleung
/include/device.h @wentongwu @nashif
/include/devicetree.h @galak
/include/display/ @vanwinkeljan
@ -426,6 +427,7 @@
/samples/userspace/ @andrewboie
/scripts/coccicheck @himanshujha199640 @JuliaLawall
/scripts/coccinelle/ @himanshujha199640 @JuliaLawall
/scripts/coredump/ @dcpleung
/scripts/kconfig/ @ulfalizer
/scripts/sanity_chk/expr_parser.py @nashif
/scripts/gen_app_partitions.py @andrewboie
@ -460,6 +462,7 @@
/subsys/canbus/ @alexanderwachter
/subsys/cpp/ @pabigot @vanwinkeljan
/subsys/debug/ @nashif
/subsys/debug/coredump/ @dcpleung
/subsys/dfu/ @nvlsianpu
/subsys/tracing/ @nashif @wentongwu
/subsys/debug/asan_hacks.c @vanwinkeljan @aescolar @daor-oti

View file

@ -392,6 +392,9 @@ config ARCH_HAS_RAMFUNC_SUPPORT
config ARCH_HAS_NESTED_EXCEPTION_DETECTION
bool
config ARCH_SUPPORTS_COREDUMP
bool
#
# Other architecture related options
#

View file

@ -0,0 +1,347 @@
.. _coredump:
Core Dump
#########
The core dump module enables dumping the CPU registers and memory content
for offline debugging. This module is called when fatal error is
encountered, and the data is printed or stored according to which backends
are enabled.
Configuration
*************
Configure this module using the following options.
* ``DEBUG_COREDUMP``: enable the module.
Here are the options to enable output backends for core dump:
* ``DEBUG_COREDUMP_BACKEND_LOGGING``: use log module for core dump output.
* ``DEBUG_COREDUMP_BACKEND_NULL``: fallback core dump backend if other
backends cannot be enabled. All output is sent to null.
Here are the choices regarding memory dump:
* ``DEBUG_COREDUMP_MEMORY_DUMP_MIN```: only dumps the stack of the exception
thread, its thread struct, and some other bare minimal data to support
walking the stack in debugger. Use this only if absolute minimum of data
dump is desired.
Usage
*****
When the core dump module is enabled, during fatal error, CPU registers
and memory content are being printed or stored according to which backends
are enabled. This core dump data can fed into a custom made GDB server as
a remote target for GDB (and other GDB compatible debuggers). CPU registers,
memory content and stack can be examined in the debugger.
This usually involves the following steps:
1. Get the core dump log from the device depending on enabled backends.
For example, if the log module backend is used, get the log output
from the log module backend.
2. Convert the core dump log into a binary format that can be parsed by
the GDB server. For example,
:zephyr_file:`scripts/coredump/coredump_serial_log_parser.py` can be used
to convert the serial console log into a binary file.
3. Start the custom GDB server using the script
:zephyr_file:`scripts/coredump/coredump_gdbserver.py` with the core dump
binary log file, and the Zephyr ELF file as parameters.
4. Start the debugger corresponding to the target architecture.
Example
-------
This example uses the log module backend tied to serial console.
This was done on `qemu_x86` where a null pointer was dereferenced.
This is the core dump log from the serial console, and is stored
in :file:`coredump.log`:
::
Booting from ROM..*** Booting Zephyr OS build zephyr-v2.3.0-1840-g7bba91944a63 ***
Hello World! qemu_x86
E: Page fault at address 0x0 (error code 0x2)
E: Linear address not present in page tables
E: PDE: 0x0000000000115827 Writable, User, Execute Enabled
E: PTE: Non-present
E: EAX: 0x00000000, EBX: 0x00000000, ECX: 0x00119d74, EDX: 0x000003f8
E: ESI: 0x00000000, EDI: 0x00101aa7, EBP: 0x00119d10, ESP: 0x00119d00
E: EFLAGS: 0x00000206 CS: 0x0008 CR3: 0x00119000
E: call trace:
E: EIP: 0x00100459
E: 0x00100477 (0x0)
E: 0x00100492 (0x0)
E: 0x001004c8 (0x0)
E: 0x00105465 (0x105465)
E: 0x00101abe (0x0)
E: >>> ZEPHYR FATAL ERROR 0: CPU exception on CPU 0
E: Current thread: 0x00119080 (unknown)
E: #CD:BEGIN#
E: #CD:5a4501000100050000000000
E: #CD:4101003800
E: #CD:0e0000000200000000000000749d1100f803000000000000009d1100109d1100
E: #CD:00000000a71a100059041000060200000800000000901100
E: #CD:4d010080901100e0901100
E: #CD:0100000000000000000000000180000000000000000000000000000000000000
E: #CD:00000000000000000000000000000000e364100000000000000000004c9c1100
E: #CD:000000000000000000000000b49911000004000000000000fc03000000000000
E: #CD:4d0100b4991100b49d1100
E: #CD:f8030000020000000200000002000000f8030000fd03000a02000000dc9e1100
E: #CD:149a1160fd03000002000000dc9e1100249a110087201000049f11000a000000
E: #CD:349a11000a4f1000049f11000a9e1100449a11000a8b10000200000002000000
E: #CD:449a1100388b1000049f11000a000000549a1100ad201000049f11000a000000
E: #CD:749a11000a201000049f11000a000000649a11000a201000049f11000a000000
E: #CD:749a1100e8201000049f11000a000000949a1100890b10000a0000000a000000
E: #CD:a49a1100890b10000a0000000a000000f8030000189b11000200000002000000
E: #CD:f49a1100289b11000a000000189b1100049b11009b0710000a000000289b1100
E: #CD:f49a110087201000049f110045000000f49a1100509011000a00000020901100
E: #CD:f49a110060901100049f1100ffffffff0000000000000000049f1100ffffffff
E: #CD:0000000000000000630b1000189b1100349b1100af0b1000630b1000289b1100
E: #CD:55891000789b11000000000020901100549b1100480000004a891000609b1100
E: #CD:649b1100d00b10004a891000709b110000000000609b11000a00000000000000
E: #CD:849b1100709b11004a89100000000000949b1100794a10000000000058901100
E: #CD:20901100c34a10000a00001734020000d001000000000000d49b110038000000
E: #CD:c49b110078481000b49911000004000000000000000000000c9c11000c9c1100
E: #CD:149c110000000000d49b110038000000f49b1100da481000b499110000040000
E: #CD:0e0000000200000000000000744d0100b4991100b49d1100009d1100109d1100
E: #CD:149c110099471000b4991100000400000800000000901100ad861000409c1100
E: #CD:349c1100e94710008090110000000000349c1100b64710008086100045000000
E: #CD:849c11002d53100000000000d09c11008090110020861000f5ffffff8c9c1100
E: #CD:000000000000000000000000a71a1000a49c1100020200008090110000000000
E: #CD:a49c1100020200000800000000000000a49c11001937100000000000d09c1100
E: #CD:0c9d0000bc9c0000b49d1100b4991100c49c1100ae37100000000000d09c1100
E: #CD:0800000000000000c888100000000000109d11005d031000d09c1100009d1100
E: #CD:109d11000000000000000000a71a1000f803000000000000749d110002000000
E: #CD:5904100008000000060200000e0000000202000002020000000000002c9d1100
E: #CD:7704100000000000d00b1000c9881000549d110000000000489d110092041000
E: #CD:00000000689d1100549d11000000000000000000689d1100c804100000000000
E: #CD:c0881000000000007c9d110000000000749d11007c9d11006554100065541000
E: #CD:00000000000000009c9d1100be1a100000000000000000000000000038041000
E: #CD:08000000020200000000000000000000f4531000000000000000000000000000
E: #CD:END#
E: Halting system
1. Run the core dump serial log converter:
.. code-block:: console
./scripts/coredump/coredump_serial_log_parser.py coredump.log coredump.bin
2. Start the custom GDB server:
.. code-block:: console
./scripts/coredump/coredump_gdbserver.py build/zephyr/zephyr.elf coredump.bin
3. Start GDB:
.. code-block:: console
<path to SDK>/x86_64-zephyr-elf/bin/x86_64-zephyr-elf-gdb build/zephyr/zephyr.elf
4. Inside GDB, connect to the GDB server via port 1234:
.. code-block:: console
(gdb) target remote localhost:1234
5. Examine the CPU registers:
.. code-block:: console
(gdb) info registers
Output from GDB:
::
eax 0x0 0
ecx 0x119d74 1154420
edx 0x3f8 1016
ebx 0x0 0
esp 0x119d00 0x119d00 <z_main_stack+844>
ebp 0x119d10 0x119d10 <z_main_stack+860>
esi 0x0 0
edi 0x101aa7 1055399
eip 0x100459 0x100459 <func_3+16>
eflags 0x206 [ PF IF ]
cs 0x8 8
ss <unavailable>
ds <unavailable>
es <unavailable>
fs <unavailable>
gs <unavailable>
6. Examine the backtrace:
.. code-block:: console
(gdb) bt
Output from GDB:
::
#0 0x00100459 in func_3 (addr=0x0) at zephyr/rtos/zephyr/samples/hello_world/src/main.c:14
#1 0x00100477 in func_2 (addr=0x0) at zephyr/rtos/zephyr/samples/hello_world/src/main.c:21
#2 0x00100492 in func_1 (addr=0x0) at zephyr/rtos/zephyr/samples/hello_world/src/main.c:28
#3 0x001004c8 in main () at zephyr/rtos/zephyr/samples/hello_world/src/main.c:42
File Format
***********
The core dump binary file consists of one file header, one
architecture-specific block, and multiple memory blocks. All numbers in
the headers below are little endian.
File Header
-----------
The file header consists of the following fields:
.. list-table:: Core dump binary file header
:widths: 2 1 7
:header-rows: 1
* - Field
- Data Type
- Description
* - ID
- ``char[2]``
- ``Z``, ``E`` as identifier of file.
* - Header version
- ``uint16_t``
- Identify the version of the header. This needs to be incremented
whenever the header struct is modified. This allows parser to
reject older header versions so it will not incorrectly parse
the header.
* - Target code
- ``uint16_t``
- Indicate which target (e.g. architecture or SoC) so the parser
can instantiate the correct register block parser.
* - Pointer size
- 'uint8_t'
- Size of ``uintptr_t`` in power of 2. (e.g. 5 for 32-bit,
6 for 64-bit). This is needed to accommodate 32-bit and 64-bit
target in parsing the memory block addresses.
* - Flags
- ``uint8_t``
-
* - Fatal error reason
- ``unsigned int``
- Reason for the fatal error, as the same in
``enum k_fatal_error_reason`` defined in
:zephyr_file:`include/fatal.h`
Architecture-specific Block
---------------------------
The architecture-specific block contains the byte stream of data specific
to the target architecture (e.g. CPU registers)
.. list-table:: Architecture-specific Block
:widths: 2 1 7
:header-rows: 1
* - Field
- Data Type
- Description
* - ID
- ``char``
- ``A`` to indiciate this is a architecture-specific block.
* - Header version
- ``uint16_t``
- Identify the version of this block. To be interpreted by the target
architecture specific block parser.
* - Number of bytes
- ``uint16_t``
- Number of bytes following the header which contains the byte stream
for target data. The format of the byte stream is specific to
the target and is only being parsed by the target parser.
* - Register byte stream
- ``uint8_t[]``
- Contains target architecture specific data.
Memory Block
------------
The memory block contains the start and end addresses and the data within
the memory region.
.. list-table:: Memory Block
:widths: 2 1 7
:header-rows: 1
* - Field
- Data Type
- Description
* - ID
- ``char``
- ``M`` to indiciate this is a memory block.
* - Header version
- ``uint16_t``
- Identify the version of the header. This needs to be incremented
whenever the header struct is modified. This allows parser to
reject older header versions so it will not incorrectly parse
the header.
* - Start address
- ``uintptr_t``
- The start address of the memory region.
* - End address
- ``uintptr_t``
- The end address of the memory region.
* - Memory byte stream
- ``uint8_t[]``
- Contains the memory content between the start and end addresses.
Adding New Target
*****************
The architecture-specific block is target specific and requires new
dumping routine and parser for new targets. To add a new target,
the following needs to be done:
#. Add a new target code to the ``enum z_coredump_tgt_code`` in
:zephyr_file:`include/debug/coredump.h`.
#. Implement :c:func:`arch_coredump_tgt_code_get` simply to return
the newly introducted target code.
#. Implement :c:func:`arch_coredump_info_dump` to construct
a target architecture block and call :c:func:`z_coredump_buffer_output`
to output the block to core dump backend.
#. Add a parser to the core dump GDB stub scripts under
``scripts/coredump/gdbstubs/``
#. Extends the ``gdbstubs.gdbstub.GdbStub`` class.
#. During ``__init__``, store the GDB signal corresponding to
the exception reason in ``self.gdb_signal``.
#. Parse the architecture-specific block from
``self.logfile.get_arch_data()``. This needs to match the format
as implemented in step 3 (inside :c:func:`arch_coredump_info_dump`).
#. Implement the abstract method ``handle_register_group_read_packet``
where it returns the register group as GDB expected. Refer to
GDB's code and documentation on what it is expecting for
the new target.
#. Optionally implement ``handle_register_single_read_packet``
for registers not covered in the ``g`` packet.
#. Extend ``get_gdbstub()`` in
:zephyr_file:`scripts/coredump/gdbstubs/__init__.py` to return
the newly implemented GDB stub.
API documentation
*****************
.. doxygengroup:: coredump_apis
:project: Zephyr
.. doxygengroup:: arch-coredump
:project: Zephyr

View file

@ -9,3 +9,4 @@ Debugging
host-tools.rst
probes.rst
thread-analyzer.rst
coredump.rst

141
include/debug/coredump.h Normal file
View file

@ -0,0 +1,141 @@
/*
* Copyright (c) 2020 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef ZEPHYR_INCLUDE_DEBUG_COREDUMP_H_
#define ZEPHYR_INCLUDE_DEBUG_COREDUMP_H_
#ifdef CONFIG_DEBUG_COREDUMP
#include <toolchain.h>
#include <arch/cpu.h>
#include <sys/byteorder.h>
#define Z_COREDUMP_HDR_VER 1
#define Z_COREDUMP_ARCH_HDR_ID 'A'
#define Z_COREDUMP_MEM_HDR_ID 'M'
#define Z_COREDUMP_MEM_HDR_VER 1
/* Target code */
enum z_coredump_tgt_code {
COREDUMP_TGT_UNKNOWN = 0,
};
/* Coredump header */
struct z_coredump_hdr_t {
/* 'Z', 'E' */
char id[2];
/* Header version */
uint16_t hdr_version;
/* Target code */
uint16_t tgt_code;
/* Pointer size in Log2 */
uint8_t ptr_size_bits;
uint8_t flag;
/* Coredump Reason given */
unsigned int reason;
} __packed;
/* Architecture-specific block header */
struct z_coredump_arch_hdr_t {
/* Z_COREDUMP_ARCH_HDR_ID */
char id;
/* Header version */
uint16_t hdr_version;
/* Number of bytes in this block (excluding header) */
uint16_t num_bytes;
} __packed;
/* Memory block header */
struct z_coredump_mem_hdr_t {
/* Z_COREDUMP_MEM_HDR_ID */
char id;
/* Header version */
uint16_t hdr_version;
/* Address of start of memory region */
uintptr_t start;
/* Address of end of memory region */
uintptr_t end;
} __packed;
void z_coredump(unsigned int reason, const z_arch_esf_t *esf,
struct k_thread *thread);
void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr);
int z_coredump_buffer_output(uint8_t *buf, size_t buflen);
#else
void z_coredump(unsigned int reason, const z_arch_esf_t *esf,
struct k_thread *thread)
{
}
void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr)
{
}
int z_coredump_buffer_output(uint8_t *buf, size_t buflen)
{
return 0;
}
#endif /* CONFIG_DEBUG_COREDUMP */
/**
* @defgroup coredump_apis Coredump APIs
* @brief Coredump APIs
* @{
*/
/**
* @fn void z_coredump(unsigned int reason, const z_arch_esf_t *esf, struct k_thread *thread);
* @brief Perform coredump.
*
* Normally, this is called inside z_fatal_error() to generate coredump
* when a fatal error is encountered. This can also be called on demand
* whenever a coredump is desired.
*
* @param reason Reason for the fatal error
* @param esf Exception context
* @param thread Thread information to dump
*/
/**
* @fn void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr);
* @brief Dump memory region
*
* @param start_addr Start address of memory region to be dumped
* @param emd_addr End address of memory region to be dumped
*/
/**
* @fn int z_coredump_buffer_output(uint8_t *buf, size_t buflen);
* @brief Output the buffer via coredump
*
* This outputs the buffer of byte array to the coredump backend.
* For example, this can be called to output the coredump section
* containing registers, or a section for memory dump.
*
* @param buf Buffer to be send to coredump output
* @param buflen Buffer length
*/
/**
* @}
*/
#endif /* ZEPHYR_INCLUDE_DEBUG_COREDUMP_H_ */

View file

@ -13,6 +13,7 @@
#include <logging/log_ctrl.h>
#include <logging/log.h>
#include <fatal.h>
#include <debug/coredump.h>
LOG_MODULE_DECLARE(os);
@ -120,6 +121,8 @@ void z_fatal_error(unsigned int reason, const z_arch_esf_t *esf)
LOG_ERR("Current thread: %p (%s)", thread,
log_strdup(thread_name_get(thread)));
z_coredump(reason, esf, thread);
k_sys_fatal_error_handler(reason, esf);
/* If the system fatal error handler returns, then kill the faulting

View file

@ -299,6 +299,28 @@ static inline void arch_nop(void);
/** @} */
/**
* @defgroup arch-coredump Architecture-specific core dump APIs
* @ingroup arch-interface
* @{
*/
/**
* @brief Architecture-specific handling during coredump
*
* This dumps architecture-specific information during coredump.
*
* @param esf Exception Stack Frame (arch-specific)
*/
void arch_coredump_info_dump(const z_arch_esf_t *esf);
/**
* @brief Get the target code specified by the architecture.
*/
uint16_t arch_coredump_tgt_code_get(void);
/** @} */
/* Include arch-specific inline function implementation */
#include <kernel_arch_func.h>

View file

@ -0,0 +1,132 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import argparse
import logging
import os
import socket
import sys
from parser.log_parser import CoredumpLogFile
from parser.elf_parser import CoredumpElfFile
import gdbstubs
LOGGING_FORMAT = "[%(levelname)s][%(name)s] %(message)s"
# Only bind to local host
GDBSERVER_HOST = ""
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("elffile", help="Zephyr ELF binary")
parser.add_argument("logfile", help="Coredump binary log file")
parser.add_argument("--debug", action="store_true",
help="Print extra debugging information")
parser.add_argument("--port", type=int, default=1234,
help="GDB server port")
parser.add_argument("-v", "--verbose", action="store_true",
help="Print more information")
return parser.parse_args()
def main():
args = parse_args()
# Setup logging
logging.basicConfig(format=LOGGING_FORMAT)
# Setup logging for "parser"
logger = logging.getLogger("parser")
if args.debug:
logger.setLevel(logging.DEBUG)
elif args.verbose:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.WARNING)
# Setup logging for follow code
logger = logging.getLogger("gdbserver")
if args.debug:
logger.setLevel(logging.DEBUG)
else:
# Use INFO as default since we need to let user
# know what is going on
logger.setLevel(logging.INFO)
# Setup logging for "gdbstuc"
logger = logging.getLogger("gdbstub")
if args.debug:
logger.setLevel(logging.DEBUG)
elif args.verbose:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.WARNING)
if not os.path.isfile(args.elffile):
logger.error(f"Cannot find file {args.elffile}, exiting...")
sys.exit(1)
if not os.path.isfile(args.logfile):
logger.error(f"Cannot find file {args.logfile}, exiting...")
sys.exit(1)
logger.info(f"Log file: {args.logfile}")
logger.info(f"ELF file: {args.elffile}")
# Parse the coredump binary log file
logf = CoredumpLogFile(args.logfile)
logf.open()
if not logf.parse():
logger.error("Cannot parse log file, exiting...")
logf.close()
sys.exit(1)
# Parse ELF file for code and read-only data
elff = CoredumpElfFile(args.elffile)
elff.open()
if not elff.parse():
logger.error("Cannot parse ELF file, exiting...")
elff.close()
logf.close()
sys.exit(1)
gdbstub = gdbstubs.get_gdbstub(logf, elff)
# Start a GDB server
gdbserver = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Reuse address so we don't have to wait for socket to be
# close before we can bind to the port again
gdbserver.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
gdbserver.bind((GDBSERVER_HOST, args.port))
gdbserver.listen(1)
logger.info(f"Waiting GDB connection on port {args.port}...")
conn, remote = gdbserver.accept()
if conn:
logger.info(f"Accepted GDB connection from {remote}")
gdbstub.run(conn)
conn.close()
gdbserver.close()
logger.info("GDB session finished.")
elff.close()
logf.close()
if __name__ == "__main__":
main()

View file

@ -0,0 +1,99 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import argparse
import binascii
import sys
COREDUMP_PREFIX_STR = "#CD:"
COREDUMP_BEGIN_STR = COREDUMP_PREFIX_STR + "BEGIN#"
COREDUMP_END_STR = COREDUMP_PREFIX_STR + "END#"
COREDUMP_ERROR_STR = COREDUMP_PREFIX_STR + "ERROR CANNOT DUMP#"
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("infile", help="Serial Log File")
parser.add_argument("outfile",
help="Output file for use with coredump GDB server")
return parser.parse_args()
def main():
args = parse_args()
infile = open(args.infile, "r")
if not infile:
print(f"ERROR: Cannot open input file: {args.infile}, exiting...")
sys.exit(1)
outfile = open(args.outfile, "wb")
if not outfile:
print(f"ERROR: Cannot open output file for write: {args.outfile}, exiting...")
sys.exit(1)
print(f"Input file {args.infile}")
print(f"Output file {args.outfile}")
has_begin = False
has_end = False
has_error = False
go_parse_line = False
bytes_written = 0
for line in infile.readlines():
if line.find(COREDUMP_BEGIN_STR) >= 0:
# Found "BEGIN#" - beginning of log
has_begin = True
go_parse_line = True
continue
if line.find(COREDUMP_END_STR) >= 0:
# Found "END#" - end of log
has_end = True
go_parse_line = False
break
if line.find(COREDUMP_ERROR_STR) >= 0:
# Error was encountered during dumping:
# log is not usable
has_error = True
go_parse_line = False
break
if not go_parse_line:
continue
prefix_idx = line.find(COREDUMP_PREFIX_STR)
if prefix_idx < 0:
continue
prefix_idx += len(COREDUMP_PREFIX_STR)
hex_str = line[prefix_idx:].strip()
binary_data = binascii.unhexlify(hex_str)
outfile.write(binary_data)
bytes_written += len(binary_data)
if not has_begin:
print("ERROR: Beginning of log not found!")
elif not has_end:
print("WARN: End of log not found! Is log complete?")
elif has_error:
print("ERROR: log has error.")
else:
print(f"Bytes written {bytes_written}")
infile.close()
outfile.close()
if __name__ == "__main__":
main()

View file

@ -0,0 +1,15 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
class TgtCode:
UNKNOWN = 0
def get_gdbstub(logfile, elffile):
stub = None
tgt_code = logfile.log_hdr['tgt_code']
return stub

View file

@ -0,0 +1,5 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

View file

@ -0,0 +1,205 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import abc
import binascii
import logging
logger = logging.getLogger("gdbstub")
class GdbStub(abc.ABC):
def __init__(self, logfile, elffile):
self.logfile = logfile
self.elffile = elffile
self.socket = None
self.gdb_signal = None
mem_regions = list()
for r in logfile.get_memory_regions():
mem_regions.append(r)
for r in elffile.get_memory_regions():
mem_regions.append(r)
self.mem_regions = mem_regions
def get_gdb_packet(self):
socket = self.socket
if socket is None:
return None
data = b''
checksum = 0
# Wait for '$'
while True:
ch = socket.recv(1)
if ch == b'$':
break
# Get a full packet
while True:
ch = socket.recv(1)
if ch == b'#':
# End of packet
break
checksum += ord(ch)
data += ch
# Get checksum (2-bytes)
ch = socket.recv(2)
in_chksum = ord(binascii.unhexlify(ch))
logger.debug(f"Received GDB packet: {data}")
if (checksum % 256) == in_chksum:
# ACK
logger.debug("ACK")
socket.send(b'+')
return data
else:
# NACK
logger.debug(f"NACK (checksum {in_chksum} != {checksum}")
socket.send(b'-')
return None
def put_gdb_packet(self, data):
socket = self.socket
if socket is None:
return
checksum = 0
for d in data:
checksum += d
pkt = b'$' + data + b'#'
checksum = checksum % 256
pkt += format(checksum, "02X").encode()
logger.debug(f"Sending GDB packet: {pkt}")
socket.send(pkt)
def handle_signal_query_packet(self):
# the '?' packet
pkt = b'S'
pkt += format(self.gdb_signal, "02X").encode()
self.put_gdb_packet(pkt)
@abc.abstractmethod
def handle_register_group_read_packet(self):
# the 'g' packet for reading a group of registers
pass
def handle_register_group_write_packet(self):
# the 'G' packet for writing to a group of registers
#
# We don't support writing so return error
self.put_gdb_packet(b"E01")
def handle_register_single_read_packet(self, pkt):
# the 'p' packet for reading a single register
self.put_gdb_packet(b"E01")
def handle_register_single_write_packet(self, pkt):
# the 'P' packet for writing to registers
#
# We don't support writing so return error
self.put_gdb_packet(b"E01")
def handle_memory_read_packet(self, pkt):
# the 'm' packet for reading memory: m<addr>,<len>
def get_mem_region(addr):
for r in self.mem_regions:
if r['start'] <= addr <= r['end']:
return r
return None
# extract address and length from packet
# and convert them into usable integer values
str_addr, str_length = pkt[1:].split(b',')
s_addr = int(b'0x' + str_addr, 16)
length = int(b'0x' + str_length, 16)
# FIXME: Need more efficient way of extracting memory content
remaining = length
addr = s_addr
barray = b''
r = get_mem_region(addr)
while remaining > 0:
if addr > r['end']:
r = get_mem_region(addr)
if r is None:
barray = None
break
offset = addr - r['start']
barray += r['data'][offset:offset+1]
addr += 1
remaining -= 1
if barray is not None:
pkt = binascii.hexlify(barray)
self.put_gdb_packet(pkt)
else:
self.put_gdb_packet(b"E01")
def handle_memory_write_packet(self, pkt):
# the 'M' packet for writing to memory
#
# We don't support writing so return error
self.put_gdb_packet(b"E02")
def handle_general_query_packet(self, pkt):
self.put_gdb_packet(b'')
def run(self, socket):
self.socket = socket
while True:
pkt = self.get_gdb_packet()
if pkt is None:
continue
pkt_type = pkt[0:1]
logger.debug(f"Got packet type: {pkt_type}")
if pkt_type == b'?':
self.handle_signal_query_packet()
elif pkt_type in (b'C', b'S'):
# Continue/stepping execution, which is not supported.
# So signal exception again
self.handle_signal_query_packet()
elif pkt_type == b'g':
self.handle_register_group_read_packet()
elif pkt_type == b'G':
self.handle_register_group_write_packet()
elif pkt_type == b'p':
self.handle_register_single_read_packet(pkt)
elif pkt_type == b'P':
self.handle_register_single_write_packet(pkt)
elif pkt_type == b'm':
self.handle_memory_read_packet(pkt)
elif pkt_type == b'M':
self.handle_memory_write_packet(pkt)
elif pkt_type == b'q':
self.handle_general_query_packet(pkt)
elif pkt_type == b'k':
# GDB quits
break
else:
self.put_gdb_packet(b'')

View file

@ -0,0 +1,5 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0

View file

@ -0,0 +1,93 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import logging
import elftools
from elftools.elf.elffile import ELFFile
# ELF section flags
SHF_WRITE = 0x1
SHF_ALLOC = 0x2
SHF_EXEC = 0x4
SHF_WRITE_ALLOC = SHF_WRITE | SHF_ALLOC
SHF_ALLOC_EXEC = SHF_ALLOC | SHF_EXEC
logger = logging.getLogger("parser")
class CoredumpElfFile():
"""
Class to parse ELF file for memory content in various sections.
There are read-only sections (e.g. text and rodata) where
the memory content does not need to be dumped via coredump
and can be retrived from the ELF file.
"""
def __init__(self, elffile):
self.elffile = elffile
self.fd = None
self.elf = None
self.memory_regions = list()
def open(self):
self.fd = open(self.elffile, "rb")
self.elf = ELFFile(self.fd)
def close(self):
self.fd.close()
def get_memory_regions(self):
return self.memory_regions
def parse(self):
if self.fd is None:
self.open()
for section in self.elf.iter_sections():
# REALLY NEED to match exact type as all other sections
# (debug, text, etc.) are descendants where
# isinstance() would match.
if type(section) is not elftools.elf.sections.Section: # pylint: disable=unidiomatic-typecheck
continue
size = section['sh_size']
flags = section['sh_flags']
sec_start = section['sh_addr']
sec_end = sec_start + size - 1
store = False
sect_desc = "?"
if section['sh_type'] == 'SHT_PROGBITS':
if (flags & SHF_ALLOC_EXEC) == SHF_ALLOC_EXEC:
# Text section
store = True
sect_desc = "text"
elif (flags & SHF_WRITE_ALLOC) == SHF_WRITE_ALLOC:
# Data section
#
# Running app changes the content so no need
# to store
pass
elif (flags & SHF_ALLOC) == SHF_ALLOC:
# Read only data section
store = True
sect_desc = "read-only data"
if store:
mem_region = {"start": sec_start, "end": sec_end, "data": section.data()}
logger.info("ELF Section: 0x%x to 0x%x of size %d (%s)" %
(mem_region["start"],
mem_region["end"],
len(mem_region["data"]),
sect_desc))
self.memory_regions.append(mem_region)
return True

View file

@ -0,0 +1,167 @@
#!/usr/bin/env python3
#
# Copyright (c) 2020 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import logging
import struct
# Note: keep sync with C code
Z_COREDUMP_HDR_ID = b'ZE'
Z_COREDUMP_HDR_VER = 1
LOG_HDR_STRUCT = "<ccHHBBI"
LOG_HDR_SIZE = struct.calcsize(LOG_HDR_STRUCT)
Z_COREDUMP_ARCH_HDR_ID = b'A'
LOG_ARCH_HDR_STRUCT = "<cHH"
LOG_ARCH_HDR_SIZE = struct.calcsize(LOG_ARCH_HDR_STRUCT)
Z_COREDUMP_MEM_HDR_ID = b'M'
Z_COREDUMP_MEM_HDR_VER = 1
LOG_MEM_HDR_STRUCT = "<cH"
LOG_MEM_HDR_SIZE = struct.calcsize(LOG_MEM_HDR_STRUCT)
logger = logging.getLogger("parser")
def reason_string(reason):
# Keep sync with "enum k_fatal_error_reason"
ret = "(Unknown)"
if reason == 0:
ret = "K_ERR_CPU_EXCEPTION"
elif reason == 1:
ret = "K_ERR_SPURIOUS_IRQ"
elif reason == 2:
ret = "K_ERR_STACK_CHK_FAIL"
elif reason == 3:
ret = "K_ERR_KERNEL_OOPS"
elif reason == 4:
ret = "K_ERR_KERNEL_PANIC"
return ret
class CoredumpLogFile:
"""
Process the binary coredump file for register block
and memory blocks.
"""
def __init__(self, logfile):
self.logfile = logfile
self.fd = None
self.log_hdr = None
self.arch_data = list()
self.memory_regions = list()
def open(self):
self.fd = open(self.logfile, "rb")
def close(self):
self.fd.close()
def get_arch_data(self):
return self.arch_data
def get_memory_regions(self):
return self.memory_regions
def parse_arch_section(self):
hdr = self.fd.read(LOG_ARCH_HDR_SIZE)
_, hdr_ver, num_bytes = struct.unpack(LOG_ARCH_HDR_STRUCT, hdr)
arch_data = self.fd.read(num_bytes)
self.arch_data = {"hdr_ver" : hdr_ver, "data" : arch_data}
return True
def parse_memory_section(self):
hdr = self.fd.read(LOG_MEM_HDR_SIZE)
_, hdr_ver = struct.unpack(LOG_MEM_HDR_STRUCT, hdr)
if hdr_ver != Z_COREDUMP_MEM_HDR_VER:
logger.error(f"Memory block version: {hdr_ver}, expected {Z_COREDUMP_MEM_HDR_VER}!")
return False
# Figure out how to read the start and end addresses
ptr_fmt = None
if self.log_hdr["ptr_size"] == 64:
ptr_fmt = "QQ"
elif self.log_hdr["ptr_size"] == 32:
ptr_fmt = "II"
else:
return False
data = self.fd.read(struct.calcsize(ptr_fmt))
saddr, eaddr = struct.unpack(ptr_fmt, data)
size = eaddr - saddr
data = self.fd.read(size)
mem = {"start": saddr, "end": eaddr, "data": data}
self.memory_regions.append(mem)
logger.info("Memory: 0x%x to 0x%x of size %d" %
(saddr, eaddr, size))
return True
def parse(self):
if self.fd is None:
self.open()
hdr = self.fd.read(LOG_HDR_SIZE)
id1, id2, hdr_ver, tgt_code, ptr_size, flags, reason = struct.unpack(LOG_HDR_STRUCT, hdr)
if (id1 + id2) != Z_COREDUMP_HDR_ID:
# ID in header does not match
logger.error("Log header ID not found...")
return False
if hdr_ver != Z_COREDUMP_HDR_VER:
logger.error(f"Log version: {hdr_ver}, expected: {Z_COREDUMP_HDR_VER}!")
return False
ptr_size = 2 ** ptr_size
self.log_hdr = {
"hdr_version": hdr_ver,
"tgt_code": tgt_code,
"ptr_size": ptr_size,
"flags": flags,
"reason": reason,
}
logger.info("Reason: {0}".format(reason_string(reason)))
logger.info(f"Pointer size {ptr_size}")
del id1, id2, hdr_ver, tgt_code, ptr_size, flags, reason
while True:
section_id = self.fd.read(1)
if not section_id:
# no more data to read
break
self.fd.seek(-1, 1) # go back 1 byte
if section_id == Z_COREDUMP_ARCH_HDR_ID:
if not self.parse_arch_section():
logger.error("Cannot parse architecture section")
return False
elif section_id == Z_COREDUMP_MEM_HDR_ID:
if not self.parse_memory_section():
logger.error("Cannot parse memory section")
return False
else:
# Unknown section in log file
logger.error(f"Unknown section in log file with ID {section_id}")
return False
return True

View file

@ -14,3 +14,8 @@ zephyr_sources_ifdef(
CONFIG_THREAD_ANALYZER
thread_analyzer.c
)
add_subdirectory_ifdef(
CONFIG_DEBUG_COREDUMP
coredump
)

View file

@ -342,4 +342,6 @@ config OPENOCD_SUPPORT
OpenOCD to determine the state of running threads. (This option
selects CONFIG_THREAD_MONITOR, so all of its caveats are implied.)
rsource "coredump/Kconfig"
endmenu

View file

@ -0,0 +1,19 @@
# Copyright (c) 2020 Intel Corporation.
# SPDX-License-Identifier: Apache-2.0
zephyr_library()
zephyr_library_include_directories(
${ZEPHYR_BASE}/kernel/include
${ZEPHYR_BASE}/arch/${ARCH}/include
)
zephyr_library_sources(
coredump_core.c
coredump_memory_regions.c
)
zephyr_library_sources_ifdef(
CONFIG_DEBUG_COREDUMP_BACKEND_LOGGING
coredump_backend_logging.c
)

View file

@ -0,0 +1,50 @@
# Copyright (c) 2020 Intel Corporation.
# SPDX-License-Identifier: Apache-2.0
menuconfig DEBUG_COREDUMP
bool "Enable Core Dump"
depends on ARCH_SUPPORTS_COREDUMP
help
Enable core dump so it can be used for offline debugging.
if DEBUG_COREDUMP
choice
prompt "Coredump backend"
default DEBUG_COREDUMP_BACKEND_LOGGING if LOG
config DEBUG_COREDUMP_BACKEND_LOGGING
bool "Use Logging subsystem for coredump"
depends on LOG
help
Core dump is done via logging subsystem.
endchoice
choice
prompt "Memory dump"
default DEBUG_COREDUMP_MEMORY_DUMP_LINKER_RAM
config DEBUG_COREDUMP_MEMORY_DUMP_MIN
bool "Minimal"
select THREAD_STACK_INFO
help
Only dumps the bare minimum memory content.
For example, the thread struct and stack of
the exception thread will be dumped.
Don't use this unless you want absolutely
minimum core dump.
config DEBUG_COREDUMP_MEMORY_DUMP_LINKER_RAM
bool "RAM defined by linker section"
help
Dumps the memory region between _image_ram_start[]
and _image_ram_end[]. This includes at least data,
noinit, and BSS sections.
This is the default.
endchoice
endif # DEBUG_COREDUMP

View file

@ -0,0 +1,93 @@
/*
* Copyright (c) 2020 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <errno.h>
#include <sys/util.h>
#include <debug/coredump.h>
#include "coredump_internal.h"
#include <logging/log.h>
LOG_MODULE_DECLARE(coredump, CONFIG_KERNEL_LOG_LEVEL);
#define COREDUMP_BEGIN_STR "BEGIN#"
#define COREDUMP_END_STR "END#"
#define COREDUMP_ERROR_STR "ERROR CANNOT DUMP#"
/*
* Need to prefix coredump strings to make it easier to parse
* as log module adds its own prefixes.
*/
#define COREDUMP_PREFIX_STR "#CD:"
/* Length of buffer of printable size */
#define LOG_BUF_SZ 64
/* Length of buffer of printable size plus null character */
#define LOG_BUF_SZ_RAW (LOG_BUF_SZ + 1)
/* Log Buffer */
char log_buf[LOG_BUF_SZ_RAW];
static void coredump_logging_backend_start(void)
{
LOG_ERR(COREDUMP_PREFIX_STR COREDUMP_BEGIN_STR);
}
static void coredump_logging_backend_end(void)
{
LOG_ERR(COREDUMP_PREFIX_STR COREDUMP_END_STR);
}
static void coredump_logging_backend_error(void)
{
LOG_ERR(COREDUMP_PREFIX_STR COREDUMP_ERROR_STR);
}
static int coredump_logging_backend_buffer_output(uint8_t *buf, size_t buflen)
{
uint8_t log_ptr = 0;
size_t remaining = buflen;
size_t i = 0;
int ret = 0;
if ((buf == NULL) || (buflen == 0)) {
ret = -EINVAL;
remaining = 0;
}
while (remaining > 0) {
if (hex2char(buf[i] >> 4, &log_buf[log_ptr]) < 0) {
ret = -EINVAL;
break;
}
log_ptr++;
if (hex2char(buf[i] & 0xf, &log_buf[log_ptr]) < 0) {
ret = -EINVAL;
break;
}
log_ptr++;
i++;
remaining--;
if ((log_ptr >= LOG_BUF_SZ) || (remaining == 0)) {
log_buf[log_ptr] = '\0';
LOG_ERR(COREDUMP_PREFIX_STR "%s", log_buf);
log_ptr = 0;
}
}
return ret;
}
struct z_coredump_backend_api z_coredump_backend_logging = {
.start = coredump_logging_backend_start,
.end = coredump_logging_backend_end,
.error = coredump_logging_backend_error,
.buffer_output = coredump_logging_backend_buffer_output,
};

View file

@ -0,0 +1,186 @@
/*
* Copyright (c) 2020 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <errno.h>
#include <kernel_internal.h>
#include <toolchain.h>
#include <debug/coredump.h>
#include <sys/byteorder.h>
#include <sys/util.h>
#include "coredump_internal.h"
#if defined(CONFIG_DEBUG_COREDUMP_BACKEND_LOGGING)
extern struct z_coredump_backend_api z_coredump_backend_logging;
static struct z_coredump_backend_api
*backend_api = &z_coredump_backend_logging;
#elif defined(DEBUG_COREDUMP_BACKEND_NULL)
extern struct z_coredump_backend_api z_coredump_backend_null;
static struct z_coredump_backend_api
*backend_api = &z_coredump_backend_null;
#else
#error "Need to select a coredump backend"
#endif
static int error;
static int dump_header(unsigned int reason)
{
struct z_coredump_hdr_t hdr = {
.id = {'Z', 'E'},
.hdr_version = Z_COREDUMP_HDR_VER,
.reason = sys_cpu_to_le16(reason),
};
if (sizeof(uintptr_t) == 8) {
hdr.ptr_size_bits = 6; /* 2^6 = 64 */
} else if (sizeof(uintptr_t) == 4) {
hdr.ptr_size_bits = 5; /* 2^5 = 32 */
} else {
hdr.ptr_size_bits = 0; /* Unknown */
}
hdr.tgt_code = sys_cpu_to_le16(arch_coredump_tgt_code_get());
return backend_api->buffer_output((uint8_t *)&hdr, sizeof(hdr));
}
static void dump_thread(struct k_thread *thread)
{
#ifdef CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_MIN
uintptr_t end_addr;
/*
* When dumping minimum information,
* the current thread struct and stack need to
* to be dumped so debugger can examine them.
*/
if (thread == NULL) {
return;
}
end_addr = POINTER_TO_UINT(thread) + sizeof(*thread);
z_coredump_memory_dump(POINTER_TO_UINT(thread), end_addr);
end_addr = thread->stack_info.start + thread->stack_info.size;
z_coredump_memory_dump(thread->stack_info.start, end_addr);
#endif
}
void process_memory_region_list(void)
{
#ifdef CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_LINKER_RAM
unsigned int idx = 0;
while (true) {
struct z_coredump_memory_region_t *r =
&z_coredump_memory_regions[idx];
if (r->end == POINTER_TO_UINT(NULL)) {
break;
}
z_coredump_memory_dump(r->start, r->end);
idx++;
}
#endif
}
void z_coredump(unsigned int reason, const z_arch_esf_t *esf,
struct k_thread *thread)
{
error = 0;
z_coredump_start();
dump_header(reason);
if (esf != NULL) {
arch_coredump_info_dump(esf);
}
if (thread != NULL) {
dump_thread(thread);
}
process_memory_region_list();
if (error != 0) {
z_coredump_error();
}
z_coredump_end();
}
void z_coredump_start(void)
{
backend_api->start();
}
void z_coredump_end(void)
{
backend_api->end();
}
void z_coredump_error(void)
{
backend_api->error();
}
int z_coredump_buffer_output(uint8_t *buf, size_t buflen)
{
int ret;
/* Error encountered before, skip */
if (error != 0) {
return -EAGAIN;
}
if ((buf == NULL) || (buflen == 0)) {
ret = -EINVAL;
} else {
error = backend_api->buffer_output(buf, buflen);
ret = error;
}
return ret;
}
void z_coredump_memory_dump(uintptr_t start_addr, uintptr_t end_addr)
{
struct z_coredump_mem_hdr_t m;
size_t len;
if ((start_addr == POINTER_TO_UINT(NULL)) ||
(end_addr == POINTER_TO_UINT(NULL))) {
return;
}
if (start_addr >= end_addr) {
return;
}
len = end_addr - start_addr;
m.id = Z_COREDUMP_MEM_HDR_ID;
m.hdr_version = Z_COREDUMP_MEM_HDR_VER;
if (sizeof(uintptr_t) == 8) {
m.start = sys_cpu_to_le64(start_addr);
m.end = sys_cpu_to_le64(end_addr);
} else if (sizeof(uintptr_t) == 4) {
m.start = sys_cpu_to_le32(start_addr);
m.end = sys_cpu_to_le32(end_addr);
}
z_coredump_buffer_output((uint8_t *)&m, sizeof(m));
z_coredump_buffer_output((uint8_t *)start_addr, len);
}

View file

@ -0,0 +1,74 @@
/*
* Copyright (c) 2020 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
#ifndef DEBUG_COREDUMP_INTERNAL_H_
#define DEBUG_COREDUMP_INTERNAL_H_
#include <toolchain.h>
/**
* @cond INTERNAL_HIDDEN
*
* These are for internal use only, so skip these in
* public documentation.
*/
struct z_coredump_memory_region_t {
uintptr_t start;
uintptr_t end;
};
extern struct z_coredump_memory_region_t z_coredump_memory_regions[];
/**
* @brief Mark the start of coredump
*
* This sets up coredump subsys so coredump can be commenced.
*
* For example, backend needs to be initialized before any
* output can be stored.
*/
void z_coredump_start(void);
/**
* @brief Mark the end of coredump
*
* This tells the coredump subsys to finalize the coredump
* session.
*
* For example, backend may need to flush the output.
*/
void z_coredump_end(void);
/**
* @brief Signal to coredump subsys there is an error.
*/
void z_coredump_error(void);
typedef void (*z_coredump_backend_start_t)(void);
typedef void (*z_coredump_backend_end_t)(void);
typedef void (*z_coredump_backend_error_t)(void);
typedef int (*z_coredump_backend_buffer_output_t)(uint8_t *buf, size_t buflen);
struct z_coredump_backend_api {
/* Signal to backend of the start of coredump. */
z_coredump_backend_start_t start;
/* Signal to backend of the end of coredump. */
z_coredump_backend_end_t end;
/* Signal to backend an error has been encountered. */
z_coredump_backend_error_t error;
/* Raw buffer output */
z_coredump_backend_buffer_output_t buffer_output;
};
/**
* @endcond
*/
#endif /* DEBUG_COREDUMP_INTERNAL_H_ */

View file

@ -0,0 +1,22 @@
/*
* Copyright (c) 2020 Intel Corporation.
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <errno.h>
#include <kernel_internal.h>
#include <toolchain.h>
#include <debug/coredump.h>
#include <linker/linker-defs.h>
#include <sys/byteorder.h>
#include <sys/util.h>
#include "coredump_internal.h"
#ifdef CONFIG_DEBUG_COREDUMP_MEMORY_DUMP_LINKER_RAM
struct z_coredump_memory_region_t __weak z_coredump_memory_regions[] = {
{(uintptr_t)&_image_ram_start, (uintptr_t)&_image_ram_end},
{0, 0} /* End of list */
};
#endif