michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: cleanup imports and move size classes out

Browse source 
More python import optimizations and cleanup and move size_calc class
out into own file.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2022-06-09 13:14:07 -04:00
commit 78d9dab7f3
12 changed files with 236 additions and 223 deletions
Download patch file Download diff file Expand all files Collapse all files

0
scripts/pylib/twister/cmakecache.py → scripts/pylib/twister/twister/cmakecache.py
View file

3
scripts/pylib/twister/enviornment.py → scripts/pylib/twister/twister/enviornment.py
View file

@ -7,7 +7,6 @@
import os
import sys
from pathlib import Path
from error import TwisterRuntimeError
import json
import logging
import subprocess
@ -17,6 +16,8 @@ import re
logger = logging.getLogger('twister')
logger.setLevel(logging.DEBUG)
from twister.error import TwisterRuntimeError
ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
if not ZEPHYR_BASE:
sys.exit("$ZEPHYR_BASE environment variable undefined")

0
scripts/pylib/twister/error.py → scripts/pylib/twister/twister/error.py
View file

2
scripts/pylib/twister/handlers.py → scripts/pylib/twister/twister/handlers.py
View file

@ -15,7 +15,7 @@ import shlex
import subprocess
import threading
import select
from enviornment import ZEPHYR_BASE
from twister.enviornment import ZEPHYR_BASE
try:
import serial

4
scripts/pylib/twister/hardwaremap.py → scripts/pylib/twister/twister/hardwaremap.py
View file

@ -7,7 +7,7 @@
import os
from multiprocessing import Lock, Value
import re
from enviornment import ZEPHYR_BASE
import platform
import yaml
import scl
@ -15,6 +15,8 @@ import logging
from pathlib import Path
import expr_parser
from twister.enviornment import ZEPHYR_BASE
try:
# Use the C LibYAML parser if available, rather than the Python parser.
# It's much faster.

0
scripts/pylib/twister/mixins.py → scripts/pylib/twister/twister/mixins.py
View file

4
scripts/pylib/twister/twister/platform.py
View file

@ -5,9 +5,9 @@
# SPDX-License-Identifier: Apache-2.0
import os
from config_parser import TwisterConfigParser
import scl
from enviornment import ZEPHYR_BASE
from twister.config_parser import TwisterConfigParser
from twister.enviornment import ZEPHYR_BASE
class Platform:
"""Class representing metadata for a particular platform

0
scripts/pylib/twister/reports.py → scripts/pylib/twister/twister/reports.py
View file

214
scripts/pylib/twister/twister/size_calc.py Normal file
View file

@ -0,0 +1,214 @@
#!/usr/bin/env python3
# vim: set syntax=python ts=4 :
#
# Copyright (c) 2018 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import subprocess
from twister.error import TwisterRuntimeError
class SizeCalculator:
alloc_sections = [
"bss",
"noinit",
"app_bss",
"app_noinit",
"ccm_bss",
"ccm_noinit"
]
rw_sections = [
"datas",
"initlevel",
"exceptions",
"initshell",
"_static_thread_data_area",
"k_timer_area",
"k_mem_slab_area",
"k_mem_pool_area",
"sw_isr_table",
"k_sem_area",
"k_mutex_area",
"app_shmem_regions",
"_k_fifo_area",
"_k_lifo_area",
"k_stack_area",
"k_msgq_area",
"k_mbox_area",
"k_pipe_area",
"net_if_area",
"net_if_dev_area",
"net_l2_area",
"net_l2_data",
"k_queue_area",
"_net_buf_pool_area",
"app_datas",
"kobject_data",
"mmu_tables",
"app_pad",
"priv_stacks",
"ccm_data",
"usb_descriptor",
"usb_data", "usb_bos_desc",
"uart_mux",
'log_backends_sections',
'log_dynamic_sections',
'log_const_sections',
"app_smem",
'shell_root_cmds_sections',
'log_const_sections',
"font_entry_sections",
"priv_stacks_noinit",
"_GCOV_BSS_SECTION_NAME",
"gcov",
"nocache",
"devices",
"k_heap_area",
]
# These get copied into RAM only on non-XIP
ro_sections = [
"rom_start",
"text",
"ctors",
"init_array",
"reset",
"z_object_assignment_area",
"rodata",
"net_l2",
"vector",
"sw_isr_table",
"settings_handler_static_area",
"bt_l2cap_fixed_chan_area",
"bt_l2cap_br_fixed_chan_area",
"bt_gatt_service_static_area",
"vectors",
"net_socket_register_area",
"net_ppp_proto",
"shell_area",
"tracing_backend_area",
"ppp_protocol_handler_area",
]
def __init__(self, filename, extra_sections):
"""Constructor
@param filename Path to the output binary
The <filename> is parsed by objdump to determine section sizes
"""
# Make sure this is an ELF binary
with open(filename, "rb") as f:
magic = f.read(4)
try:
if magic != b'\x7fELF':
raise TwisterRuntimeError("%s is not an ELF binary" % filename)
except Exception as e:
print(str(e))
sys.exit(2)
# Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
# GREP can not be used as it returns an error if the symbol is not
# found.
is_xip_command = "nm " + filename + \
" | awk '/CONFIG_XIP/ { print $3 }'"
is_xip_output = subprocess.check_output(
is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
"utf-8").strip()
try:
if is_xip_output.endswith("no symbols"):
raise TwisterRuntimeError("%s has no symbol information" % filename)
except Exception as e:
print(str(e))
sys.exit(2)
self.is_xip = (len(is_xip_output) != 0)
self.filename = filename
self.sections = []
self.rom_size = 0
self.ram_size = 0
self.extra_sections = extra_sections
self._calculate_sizes()
def get_ram_size(self):
"""Get the amount of RAM the application will use up on the device
@return amount of RAM, in bytes
"""
return self.ram_size
def get_rom_size(self):
"""Get the size of the data that this application uses on device's flash
@return amount of ROM, in bytes
"""
return self.rom_size
def unrecognized_sections(self):
"""Get a list of sections inside the binary that weren't recognized
@return list of unrecognized section names
"""
slist = []
for v in self.sections:
if not v["recognized"]:
slist.append(v["name"])
return slist
def _calculate_sizes(self):
""" Calculate RAM and ROM usage by section """
objdump_command = "objdump -h " + self.filename
objdump_output = subprocess.check_output(
objdump_command, shell=True).decode("utf-8").splitlines()
for line in objdump_output:
words = line.split()
if not words: # Skip lines that are too short
continue
index = words[0]
if not index[0].isdigit(): # Skip lines that do not start
continue # with a digit
name = words[1] # Skip lines with section names
if name[0] == '.': # starting with '.'
continue
# TODO this doesn't actually reflect the size in flash or RAM as
# it doesn't include linker-imposed padding between sections.
# It is close though.
size = int(words[2], 16)
if size == 0:
continue
load_addr = int(words[4], 16)
virt_addr = int(words[3], 16)
# Add section to memory use totals (for both non-XIP and XIP scenarios)
# Unrecognized section names are not included in the calculations.
recognized = True
if name in SizeCalculator.alloc_sections:
self.ram_size += size
stype = "alloc"
elif name in SizeCalculator.rw_sections:
self.ram_size += size
self.rom_size += size
stype = "rw"
elif name in SizeCalculator.ro_sections:
self.rom_size += size
if not self.is_xip:
self.ram_size += size
stype = "ro"
else:
stype = "unknown"
if name not in self.extra_sections:
recognized = False
self.sections.append({"name": name, "load_addr": load_addr,
"size": size, "virt_addr": virt_addr,
"type": stype, "recognized": recognized})

6
scripts/pylib/twister/testsuite.py → scripts/pylib/twister/twister/testsuite.py
View file

@ -6,9 +6,9 @@
import os
import sys
from pathlib import Path
from mixins import DisablePyTestCollectionMixin
from enviornment import canonical_zephyr_base
from error import TwisterException
from twister.mixins import DisablePyTestCollectionMixin
from twister.enviornment import canonical_zephyr_base
from twister.error import TwisterException
class TestCase(DisablePyTestCollectionMixin):

217
scripts/pylib/twister/twisterlib.py
View file

@ -16,7 +16,6 @@ import queue
import glob
import random
import logging
from pathlib import Path
from distutils.spawn import find_executable
import colorama
from colorama import Fore
@ -26,12 +25,13 @@ import json
from multiprocessing import Lock, Process, Value
from typing import List
from cmakecache import CMakeCache
from testsuite import TestCase, TestSuite
from error import TwisterRuntimeError, ConfigurationError, BuildError
from handlers import BinaryHandler, QEMUHandler, DeviceHandler
from twister.cmakecache import CMakeCache
from twister.testsuite import TestCase, TestSuite
from twister.error import TwisterRuntimeError, ConfigurationError, BuildError
from twister.handlers import BinaryHandler, QEMUHandler, DeviceHandler
from twister.platform import Platform
from twister.config_parser import TwisterConfigParser
from twister.size_calc import SizeCalculator
try:
# Use the C LibYAML parser if available, rather than the Python parser.
@ -50,7 +50,7 @@ sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts",
"python-devicetree", "src"))
from devicetree import edtlib # pylint: disable=unused-import
from enviornment import TwisterEnv, canonical_zephyr_base
from twister.enviornment import TwisterEnv, canonical_zephyr_base
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/"))
@ -187,211 +187,6 @@ class ExecutionCounter(object):
with self._total.get_lock():
return self._total.value
class SizeCalculator:
alloc_sections = [
"bss",
"noinit",
"app_bss",
"app_noinit",
"ccm_bss",
"ccm_noinit"
]
rw_sections = [
"datas",
"initlevel",
"exceptions",
"initshell",
"_static_thread_data_area",
"k_timer_area",
"k_mem_slab_area",
"k_mem_pool_area",
"sw_isr_table",
"k_sem_area",
"k_mutex_area",
"app_shmem_regions",
"_k_fifo_area",
"_k_lifo_area",
"k_stack_area",
"k_msgq_area",
"k_mbox_area",
"k_pipe_area",
"net_if_area",
"net_if_dev_area",
"net_l2_area",
"net_l2_data",
"k_queue_area",
"_net_buf_pool_area",
"app_datas",
"kobject_data",
"mmu_tables",
"app_pad",
"priv_stacks",
"ccm_data",
"usb_descriptor",
"usb_data", "usb_bos_desc",
"uart_mux",
'log_backends_sections',
'log_dynamic_sections',
'log_const_sections',
"app_smem",
'shell_root_cmds_sections',
'log_const_sections',
"font_entry_sections",
"priv_stacks_noinit",
"_GCOV_BSS_SECTION_NAME",
"gcov",
"nocache",
"devices",
"k_heap_area",
]
# These get copied into RAM only on non-XIP
ro_sections = [
"rom_start",
"text",
"ctors",
"init_array",
"reset",
"z_object_assignment_area",
"rodata",
"net_l2",
"vector",
"sw_isr_table",
"settings_handler_static_area",
"bt_l2cap_fixed_chan_area",
"bt_l2cap_br_fixed_chan_area",
"bt_gatt_service_static_area",
"vectors",
"net_socket_register_area",
"net_ppp_proto",
"shell_area",
"tracing_backend_area",
"ppp_protocol_handler_area",
]
def __init__(self, filename, extra_sections):
"""Constructor
@param filename Path to the output binary
The <filename> is parsed by objdump to determine section sizes
"""
# Make sure this is an ELF binary
with open(filename, "rb") as f:
magic = f.read(4)
try:
if magic != b'\x7fELF':
raise TwisterRuntimeError("%s is not an ELF binary" % filename)
except Exception as e:
print(str(e))
sys.exit(2)
# Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
# GREP can not be used as it returns an error if the symbol is not
# found.
is_xip_command = "nm " + filename + \
" | awk '/CONFIG_XIP/ { print $3 }'"
is_xip_output = subprocess.check_output(
is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
"utf-8").strip()
try:
if is_xip_output.endswith("no symbols"):
raise TwisterRuntimeError("%s has no symbol information" % filename)
except Exception as e:
print(str(e))
sys.exit(2)
self.is_xip = (len(is_xip_output) != 0)
self.filename = filename
self.sections = []
self.rom_size = 0
self.ram_size = 0
self.extra_sections = extra_sections
self._calculate_sizes()
def get_ram_size(self):
"""Get the amount of RAM the application will use up on the device
@return amount of RAM, in bytes
"""
return self.ram_size
def get_rom_size(self):
"""Get the size of the data that this application uses on device's flash
@return amount of ROM, in bytes
"""
return self.rom_size
def unrecognized_sections(self):
"""Get a list of sections inside the binary that weren't recognized
@return list of unrecognized section names
"""
slist = []
for v in self.sections:
if not v["recognized"]:
slist.append(v["name"])
return slist
def _calculate_sizes(self):
""" Calculate RAM and ROM usage by section """
objdump_command = "objdump -h " + self.filename
objdump_output = subprocess.check_output(
objdump_command, shell=True).decode("utf-8").splitlines()
for line in objdump_output:
words = line.split()
if not words: # Skip lines that are too short
continue
index = words[0]
if not index[0].isdigit(): # Skip lines that do not start
continue # with a digit
name = words[1] # Skip lines with section names
if name[0] == '.': # starting with '.'
continue
# TODO this doesn't actually reflect the size in flash or RAM as
# it doesn't include linker-imposed padding between sections.
# It is close though.
size = int(words[2], 16)
if size == 0:
continue
load_addr = int(words[4], 16)
virt_addr = int(words[3], 16)
# Add section to memory use totals (for both non-XIP and XIP scenarios)
# Unrecognized section names are not included in the calculations.
recognized = True
if name in SizeCalculator.alloc_sections:
self.ram_size += size
stype = "alloc"
elif name in SizeCalculator.rw_sections:
self.ram_size += size
self.rom_size += size
stype = "rw"
elif name in SizeCalculator.ro_sections:
self.rom_size += size
if not self.is_xip:
self.ram_size += size
stype = "ro"
else:
stype = "unknown"
if name not in self.extra_sections:
recognized = False
self.sections.append({"name": name, "load_addr": load_addr,
"size": size, "virt_addr": virt_addr,
"type": stype, "recognized": recognized})
class ScanPathResult:
"""Result of the TestSuite.scan_path function call.

9
scripts/twister
View file

@ -206,10 +206,10 @@ except ImportError:
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
import twisterlib
from twisterlib import TestPlan, SizeCalculator, CoverageTool, ExecutionCounter
from enviornment import TwisterEnv, canonical_zephyr_base
from reports import Reporting
from hardwaremap import HardwareMap
from twisterlib import TestPlan, CoverageTool, ExecutionCounter
from twister.enviornment import TwisterEnv, canonical_zephyr_base
from twister.reports import Reporting
from twister.hardwaremap import HardwareMap
logger = logging.getLogger('twister')
logger.setLevel(logging.DEBUG)
@ -885,6 +885,7 @@ def main():
if options.size:
for fn in options.size:
from twister.size_calc import SizeCalculator
size_report(SizeCalculator(fn, []))
return