coredump: Enable understanding of threads in scripts
Update zephyr gdb-server scripts to understand threads. Parse the kernel_thread_info out of the elf file to be used for finding offsets to data from _kernel structs or from individual threads. Update log_parser to understand latest format change, which allows for the presence of a new section, threads metadata. Update gdbstub to respond to various packets to describe the threads present in a dump, and allow switching to thread context of each thread. Signed-off-by: Mark Holden <mholden@meta.com>
This commit is contained in:
parent
c35f6d1cac
commit
0b9b33c540
3 changed files with 282 additions and 28 deletions
|
@ -5,9 +5,11 @@
|
|||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
import struct
|
||||
|
||||
import elftools
|
||||
from elftools.elf.elffile import ELFFile
|
||||
from enum import IntEnum
|
||||
|
||||
|
||||
# ELF section flags
|
||||
|
@ -17,6 +19,27 @@ SHF_EXEC = 0x4
|
|||
SHF_WRITE_ALLOC = SHF_WRITE | SHF_ALLOC
|
||||
SHF_ALLOC_EXEC = SHF_ALLOC | SHF_EXEC
|
||||
|
||||
# Must match enum in thread_info.c
|
||||
class ThreadInfoOffset(IntEnum):
|
||||
THREAD_INFO_OFFSET_VERSION = 0
|
||||
THREAD_INFO_OFFSET_K_CURR_THREAD = 1
|
||||
THREAD_INFO_OFFSET_K_THREADS = 2
|
||||
THREAD_INFO_OFFSET_T_ENTRY = 3
|
||||
THREAD_INFO_OFFSET_T_NEXT_THREAD = 4
|
||||
THREAD_INFO_OFFSET_T_STATE = 5
|
||||
THREAD_INFO_OFFSET_T_USER_OPTIONS = 6
|
||||
THREAD_INFO_OFFSET_T_PRIO = 7
|
||||
THREAD_INFO_OFFSET_T_STACK_PTR = 8
|
||||
THREAD_INFO_OFFSET_T_NAME = 9
|
||||
THREAD_INFO_OFFSET_T_ARCH = 10
|
||||
THREAD_INFO_OFFSET_T_PREEMPT_FLOAT = 11
|
||||
THREAD_INFO_OFFSET_T_COOP_FLOAT = 12
|
||||
THREAD_INFO_OFFSET_T_ARM_EXC_RETURN = 13
|
||||
THREAD_INFO_OFFSET_T_ARC_RELINQUISH_CAUSE = 14
|
||||
|
||||
def __int__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
logger = logging.getLogger("parser")
|
||||
|
||||
|
@ -34,6 +57,9 @@ class CoredumpElfFile():
|
|||
self.fd = None
|
||||
self.elf = None
|
||||
self.memory_regions = list()
|
||||
self.kernel_thread_info_offsets = None
|
||||
self.kernel_thread_info_num_offsets = None
|
||||
self.kernel_thread_info_size_t_size = None
|
||||
|
||||
def open(self):
|
||||
self.fd = open(self.elffile, "rb")
|
||||
|
@ -45,11 +71,35 @@ class CoredumpElfFile():
|
|||
def get_memory_regions(self):
|
||||
return self.memory_regions
|
||||
|
||||
def get_kernel_thread_info_size_t_size(self):
|
||||
return self.kernel_thread_info_size_t_size
|
||||
|
||||
def has_kernel_thread_info(self):
|
||||
return self.kernel_thread_info_offsets is not None
|
||||
|
||||
def get_kernel_thread_info_offset(self, thread_info_offset_index):
|
||||
if self.has_kernel_thread_info() and thread_info_offset_index <= ThreadInfoOffset.THREAD_INFO_OFFSET_T_ARC_RELINQUISH_CAUSE:
|
||||
return self.kernel_thread_info_offsets[thread_info_offset_index]
|
||||
else:
|
||||
return None
|
||||
|
||||
def parse(self):
|
||||
if self.fd is None:
|
||||
self.open()
|
||||
|
||||
kernel_thread_info_offsets_segment = None
|
||||
kernel_thread_info_num_offsets_segment = None
|
||||
_kernel_thread_info_offsets = None
|
||||
_kernel_thread_info_num_offsets = None
|
||||
_kernel_thread_info_size_t_size = None
|
||||
|
||||
for section in self.elf.iter_sections():
|
||||
# Find symbols for _kernel_thread_info data
|
||||
if isinstance(section, elftools.elf.sections.SymbolTableSection):
|
||||
_kernel_thread_info_offsets = section.get_symbol_by_name("_kernel_thread_info_offsets")
|
||||
_kernel_thread_info_num_offsets = section.get_symbol_by_name("_kernel_thread_info_num_offsets")
|
||||
_kernel_thread_info_size_t_size = section.get_symbol_by_name("_kernel_thread_info_size_t_size")
|
||||
|
||||
# REALLY NEED to match exact type as all other sections
|
||||
# (debug, text, etc.) are descendants where
|
||||
# isinstance() would match.
|
||||
|
@ -90,4 +140,48 @@ class CoredumpElfFile():
|
|||
|
||||
self.memory_regions.append(mem_region)
|
||||
|
||||
if _kernel_thread_info_size_t_size is not None and \
|
||||
_kernel_thread_info_num_offsets is not None and \
|
||||
_kernel_thread_info_offsets is not None:
|
||||
for seg in self.elf.iter_segments():
|
||||
if seg.header['p_type'] != 'PT_LOAD':
|
||||
continue
|
||||
|
||||
# Store segment of kernel_thread_info_offsets
|
||||
info_offsets_symbol = _kernel_thread_info_offsets[0]
|
||||
if info_offsets_symbol['st_value'] >= seg['p_vaddr'] and info_offsets_symbol['st_value'] < seg['p_vaddr'] + seg['p_filesz']:
|
||||
kernel_thread_info_offsets_segment = seg
|
||||
|
||||
# Store segment of kernel_thread_info_num_offsets
|
||||
num_offsets_symbol = _kernel_thread_info_num_offsets[0]
|
||||
if num_offsets_symbol['st_value'] >= seg['p_vaddr'] and num_offsets_symbol['st_value'] < seg['p_vaddr'] + seg['p_filesz']:
|
||||
kernel_thread_info_num_offsets_segment = seg
|
||||
|
||||
# Read and store size_t size
|
||||
size_t_size_symbol = _kernel_thread_info_size_t_size[0]
|
||||
if size_t_size_symbol['st_value'] >= seg['p_vaddr'] and size_t_size_symbol['st_value'] < seg['p_vaddr'] + seg['p_filesz']:
|
||||
offset = size_t_size_symbol['st_value'] - seg['p_vaddr'] + seg['p_offset']
|
||||
self.elf.stream.seek(offset)
|
||||
self.kernel_thread_info_size_t_size = struct.unpack('B', self.elf.stream.read(size_t_size_symbol['st_size']))[0]
|
||||
|
||||
struct_format = "I"
|
||||
if self.kernel_thread_info_size_t_size == 8:
|
||||
struct_format = "Q"
|
||||
|
||||
# Read and store count of offset values
|
||||
num_offsets_symbol = _kernel_thread_info_num_offsets[0]
|
||||
offset = num_offsets_symbol['st_value'] - kernel_thread_info_num_offsets_segment['p_vaddr'] + kernel_thread_info_num_offsets_segment['p_offset']
|
||||
self.elf.stream.seek(offset)
|
||||
self.kernel_thread_info_num_offsets = struct.unpack(struct_format, self.elf.stream.read(num_offsets_symbol['st_size']))[0]
|
||||
|
||||
array_format = ""
|
||||
for _ in range(self.kernel_thread_info_num_offsets):
|
||||
array_format = array_format + struct_format
|
||||
|
||||
# Read and store array of offset values
|
||||
info_offsets_symbol = _kernel_thread_info_offsets[0]
|
||||
offset = info_offsets_symbol['st_value'] - kernel_thread_info_offsets_segment['p_vaddr'] + kernel_thread_info_offsets_segment['p_offset']
|
||||
self.elf.stream.seek(offset)
|
||||
self.kernel_thread_info_offsets = struct.unpack(array_format, self.elf.stream.read(info_offsets_symbol['st_size']))
|
||||
|
||||
return True
|
||||
|
|
|
@ -10,7 +10,7 @@ import struct
|
|||
|
||||
# Note: keep sync with C code
|
||||
COREDUMP_HDR_ID = b'ZE'
|
||||
COREDUMP_HDR_VER = 1
|
||||
COREDUMP_HDR_VER = 2
|
||||
LOG_HDR_STRUCT = "<ccHHBBI"
|
||||
LOG_HDR_SIZE = struct.calcsize(LOG_HDR_STRUCT)
|
||||
|
||||
|
@ -18,6 +18,10 @@ COREDUMP_ARCH_HDR_ID = b'A'
|
|||
LOG_ARCH_HDR_STRUCT = "<cHH"
|
||||
LOG_ARCH_HDR_SIZE = struct.calcsize(LOG_ARCH_HDR_STRUCT)
|
||||
|
||||
COREDUMP_THREADS_META_HDR_ID = b'T'
|
||||
LOG_THREADS_META_HDR_STRUCT = "<cHH"
|
||||
LOG_THREADS_META_HDR_SIZE = struct.calcsize(LOG_THREADS_META_HDR_STRUCT)
|
||||
|
||||
COREDUMP_MEM_HDR_ID = b'M'
|
||||
COREDUMP_MEM_HDR_VER = 1
|
||||
LOG_MEM_HDR_STRUCT = "<cH"
|
||||
|
@ -71,6 +75,9 @@ class CoredumpLogFile:
|
|||
def get_memory_regions(self):
|
||||
return self.memory_regions
|
||||
|
||||
def get_threads_metadata(self):
|
||||
return self.threads_metadata
|
||||
|
||||
def parse_arch_section(self):
|
||||
hdr = self.fd.read(LOG_ARCH_HDR_SIZE)
|
||||
_, hdr_ver, num_bytes = struct.unpack(LOG_ARCH_HDR_STRUCT, hdr)
|
||||
|
@ -81,6 +88,16 @@ class CoredumpLogFile:
|
|||
|
||||
return True
|
||||
|
||||
def parse_threads_metadata_section(self):
|
||||
hdr = self.fd.read(LOG_THREADS_META_HDR_SIZE)
|
||||
_, hdr_ver, num_bytes = struct.unpack(LOG_THREADS_META_HDR_STRUCT, hdr)
|
||||
|
||||
data = self.fd.read(num_bytes)
|
||||
|
||||
self.threads_metadata = {"hdr_ver" : hdr_ver, "data" : data}
|
||||
|
||||
return True
|
||||
|
||||
def parse_memory_section(self):
|
||||
hdr = self.fd.read(LOG_MEM_HDR_SIZE)
|
||||
_, hdr_ver = struct.unpack(LOG_MEM_HDR_STRUCT, hdr)
|
||||
|
@ -125,7 +142,7 @@ class CoredumpLogFile:
|
|||
logger.error("Log header ID not found...")
|
||||
return False
|
||||
|
||||
if hdr_ver != COREDUMP_HDR_VER:
|
||||
if hdr_ver > COREDUMP_HDR_VER:
|
||||
logger.error(f"Log version: {hdr_ver}, expected: {COREDUMP_HDR_VER}!")
|
||||
return False
|
||||
|
||||
|
@ -155,6 +172,10 @@ class CoredumpLogFile:
|
|||
if not self.parse_arch_section():
|
||||
logger.error("Cannot parse architecture section")
|
||||
return False
|
||||
elif section_id == COREDUMP_THREADS_META_HDR_ID:
|
||||
if not self.parse_threads_metadata_section():
|
||||
logger.error("Cannot parse threads metadata section")
|
||||
return False
|
||||
elif section_id == COREDUMP_MEM_HDR_ID:
|
||||
if not self.parse_memory_section():
|
||||
logger.error("Cannot parse memory section")
|
||||
|
|
|
@ -8,6 +8,8 @@ import abc
|
|||
import binascii
|
||||
import logging
|
||||
|
||||
from coredump_parser.elf_parser import ThreadInfoOffset
|
||||
|
||||
|
||||
logger = logging.getLogger("gdbstub")
|
||||
|
||||
|
@ -18,6 +20,8 @@ class GdbStub(abc.ABC):
|
|||
self.elffile = elffile
|
||||
self.socket = None
|
||||
self.gdb_signal = None
|
||||
self.thread_ptrs = list()
|
||||
self.selected_thread = 0
|
||||
|
||||
mem_regions = list()
|
||||
|
||||
|
@ -89,6 +93,36 @@ class GdbStub(abc.ABC):
|
|||
|
||||
socket.send(pkt)
|
||||
|
||||
def get_memory(self, start_address, length):
|
||||
def get_mem_region(addr):
|
||||
for r in self.mem_regions:
|
||||
if r['start'] <= addr < r['end']:
|
||||
return r
|
||||
|
||||
return None
|
||||
|
||||
# FIXME: Need more efficient way of extracting memory content
|
||||
remaining = length
|
||||
addr = start_address
|
||||
barray = b''
|
||||
r = get_mem_region(addr)
|
||||
while remaining > 0:
|
||||
if r is None:
|
||||
barray = None
|
||||
break
|
||||
|
||||
if addr > r['end']:
|
||||
r = get_mem_region(addr)
|
||||
continue
|
||||
|
||||
offset = addr - r['start']
|
||||
barray += r['data'][offset:offset+1]
|
||||
|
||||
addr += 1
|
||||
remaining -= 1
|
||||
|
||||
return barray
|
||||
|
||||
def handle_signal_query_packet(self):
|
||||
# the '?' packet
|
||||
pkt = b'S'
|
||||
|
@ -120,38 +154,13 @@ class GdbStub(abc.ABC):
|
|||
def handle_memory_read_packet(self, pkt):
|
||||
# the 'm' packet for reading memory: m<addr>,<len>
|
||||
|
||||
def get_mem_region(addr):
|
||||
for r in self.mem_regions:
|
||||
if r['start'] <= addr < r['end']:
|
||||
return r
|
||||
|
||||
return None
|
||||
|
||||
# extract address and length from packet
|
||||
# and convert them into usable integer values
|
||||
str_addr, str_length = pkt[1:].split(b',')
|
||||
s_addr = int(b'0x' + str_addr, 16)
|
||||
length = int(b'0x' + str_length, 16)
|
||||
|
||||
# FIXME: Need more efficient way of extracting memory content
|
||||
remaining = length
|
||||
addr = s_addr
|
||||
barray = b''
|
||||
r = get_mem_region(addr)
|
||||
while remaining > 0:
|
||||
if r is None:
|
||||
barray = None
|
||||
break
|
||||
|
||||
if addr > r['end']:
|
||||
r = get_mem_region(addr)
|
||||
continue
|
||||
|
||||
offset = addr - r['start']
|
||||
barray += r['data'][offset:offset+1]
|
||||
|
||||
addr += 1
|
||||
remaining -= 1
|
||||
barray = self.get_memory(s_addr, length)
|
||||
|
||||
if barray is not None:
|
||||
pkt = binascii.hexlify(barray)
|
||||
|
@ -166,8 +175,134 @@ class GdbStub(abc.ABC):
|
|||
self.put_gdb_packet(b"E02")
|
||||
|
||||
def handle_general_query_packet(self, pkt):
|
||||
if self.arch_supports_thread_operations() and self.elffile.has_kernel_thread_info():
|
||||
# For packets qfThreadInfo/qsThreadInfo, obtain a list of all active thread IDs
|
||||
if pkt[0:12] == b"qfThreadInfo":
|
||||
threads_metadata_data = self.logfile.get_threads_metadata()["data"]
|
||||
size_t_size = self.elffile.get_kernel_thread_info_size_t_size()
|
||||
|
||||
# First, find and store the thread that _kernel considers current
|
||||
k_curr_thread_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_K_CURR_THREAD)
|
||||
curr_thread_ptr_bytes = threads_metadata_data[k_curr_thread_offset:(k_curr_thread_offset + size_t_size)]
|
||||
curr_thread_ptr = int.from_bytes(curr_thread_ptr_bytes, "little")
|
||||
self.thread_ptrs.append(curr_thread_ptr)
|
||||
|
||||
thread_count = 1
|
||||
response = b"m1"
|
||||
|
||||
# Next, find the pointer to the linked list of threads in the _kernel struct
|
||||
k_threads_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_K_THREADS)
|
||||
thread_ptr_bytes = threads_metadata_data[k_threads_offset:(k_threads_offset + size_t_size)]
|
||||
thread_ptr = int.from_bytes(thread_ptr_bytes, "little")
|
||||
|
||||
if thread_ptr != curr_thread_ptr:
|
||||
self.thread_ptrs.append(thread_ptr)
|
||||
thread_count += 1
|
||||
response += b"," + bytes(str(thread_count), 'ascii')
|
||||
|
||||
# Next walk the linked list, counting the number of threads and construct the response for qfThreadInfo along the way
|
||||
t_next_thread_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_T_NEXT_THREAD)
|
||||
while thread_ptr is not None:
|
||||
thread_ptr_bytes = self.get_memory(thread_ptr + t_next_thread_offset, size_t_size)
|
||||
|
||||
if thread_ptr_bytes is not None:
|
||||
thread_ptr = int.from_bytes(thread_ptr_bytes, "little")
|
||||
if thread_ptr == 0:
|
||||
thread_ptr = None
|
||||
continue
|
||||
|
||||
if thread_ptr != curr_thread_ptr:
|
||||
self.thread_ptrs.append(thread_ptr)
|
||||
thread_count += 1
|
||||
response += b"," + bytes(f'{thread_count:x}', 'ascii')
|
||||
else:
|
||||
thread_ptr = None
|
||||
|
||||
self.put_gdb_packet(response)
|
||||
elif pkt[0:12] == b"qsThreadInfo":
|
||||
self.put_gdb_packet(b"l")
|
||||
|
||||
# For qThreadExtraInfo, obtain a printable string description of thread attributes for the provided thread
|
||||
elif pkt[0:16] == b"qThreadExtraInfo":
|
||||
thread_info_bytes = b''
|
||||
|
||||
thread_index_str = ''
|
||||
for n in range(17, len(pkt)):
|
||||
thread_index_str += chr(pkt[n])
|
||||
|
||||
thread_id = int(thread_index_str, 16)
|
||||
if len(self.thread_ptrs) > thread_id:
|
||||
thread_info_bytes += b'name: '
|
||||
thread_ptr = self.thread_ptrs[thread_id - 1]
|
||||
t_name_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_T_NAME)
|
||||
|
||||
thread_name_next_byte = self.get_memory(thread_ptr + t_name_offset, 1)
|
||||
index = 0
|
||||
while (thread_name_next_byte is not None) and (thread_name_next_byte != b'\x00'):
|
||||
thread_info_bytes += thread_name_next_byte
|
||||
|
||||
index += 1
|
||||
thread_name_next_byte = self.get_memory(thread_ptr + t_name_offset + index, 1)
|
||||
|
||||
t_state_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_T_STATE)
|
||||
thread_state_byte = self.get_memory(thread_ptr + t_state_offset, 1)
|
||||
if thread_state_byte is not None:
|
||||
thread_state = int.from_bytes(thread_state_byte, "little")
|
||||
thread_info_bytes += b', state: ' + bytes(hex(thread_state), 'ascii')
|
||||
|
||||
t_user_options_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_T_USER_OPTIONS)
|
||||
thread_user_options_byte = self.get_memory(thread_ptr + t_user_options_offset, 1)
|
||||
if thread_user_options_byte is not None:
|
||||
thread_user_options = int.from_bytes(thread_user_options_byte, "little")
|
||||
thread_info_bytes += b', user_options: ' + bytes(hex(thread_user_options), 'ascii')
|
||||
|
||||
t_prio_offset = self.elffile.get_kernel_thread_info_offset(ThreadInfoOffset.THREAD_INFO_OFFSET_T_PRIO)
|
||||
thread_prio_byte = self.get_memory(thread_ptr + t_prio_offset, 1)
|
||||
if thread_prio_byte is not None:
|
||||
thread_prio = int.from_bytes(thread_prio_byte, "little")
|
||||
thread_info_bytes += b', prio: ' + bytes(hex(thread_prio), 'ascii')
|
||||
|
||||
self.put_gdb_packet(binascii.hexlify(thread_info_bytes))
|
||||
else:
|
||||
self.put_gdb_packet(b'')
|
||||
else:
|
||||
self.put_gdb_packet(b'')
|
||||
|
||||
def arch_supports_thread_operations(self):
|
||||
return False
|
||||
|
||||
def handle_thread_alive_packet(self, pkt):
|
||||
# the 'T' packet for finding out if a thread is alive.
|
||||
if self.arch_supports_thread_operations() and self.elffile.has_kernel_thread_info():
|
||||
# Reply OK to report thread alive, allowing GDB to perform other thread operations
|
||||
self.put_gdb_packet(b'OK')
|
||||
else:
|
||||
self.put_gdb_packet(b'')
|
||||
|
||||
def handle_thread_register_group_read_packet(self):
|
||||
self.put_gdb_packet(b'')
|
||||
|
||||
def handle_thread_op_packet(self, pkt):
|
||||
# the 'H' packet for setting thread for subsequent operations.
|
||||
if self.arch_supports_thread_operations() and self.elffile.has_kernel_thread_info():
|
||||
if pkt[0:2] == b"Hg":
|
||||
thread_index_str = ''
|
||||
for n in range(2, len(pkt)):
|
||||
thread_index_str += chr(pkt[n])
|
||||
|
||||
# Thread-id of '0' indicates an arbitrary process or thread
|
||||
if thread_index_str in ('0', ''):
|
||||
self.selected_thread = 0
|
||||
self.handle_register_group_read_packet()
|
||||
return
|
||||
|
||||
self.selected_thread = int(thread_index_str, 16) - 1
|
||||
self.handle_thread_register_group_read_packet()
|
||||
else:
|
||||
self.put_gdb_packet(b'')
|
||||
else:
|
||||
self.put_gdb_packet(b'')
|
||||
|
||||
def run(self, socket):
|
||||
self.socket = socket
|
||||
|
||||
|
@ -199,6 +334,10 @@ class GdbStub(abc.ABC):
|
|||
self.handle_memory_write_packet(pkt)
|
||||
elif pkt_type == b'q':
|
||||
self.handle_general_query_packet(pkt)
|
||||
elif pkt_type == b'T':
|
||||
self.handle_thread_alive_packet(pkt)
|
||||
elif pkt_type == b'H':
|
||||
self.handle_thread_op_packet(pkt)
|
||||
elif pkt_type == b'k':
|
||||
# GDB quits
|
||||
break
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue