michaelh/zephyr
1
0
Fork 0
Code Releases Activity

twister: measure build time and report it in json output

Browse source 
Measure both cmake and make/ninja build times and log them in debug mode
and additionally put the result in the json report for tracking build
times.

Cleanup what build/cmake function return and remove unused keys in the
result map.

Remove some excessive logging of launched jobs or default platforms that
gets in the way when in debug mode.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2023-11-22 16:30:10 -05:00
commit 2ba6bcf906
4 changed files with 38 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

1
scripts/pylib/twister/twisterlib/reports.py
View file

@ -304,6 +304,7 @@ class Reporting:
if instance.status is not None: if instance.status is not None:
suite["execution_time"] = f"{float(handler_time):.2f}" suite["execution_time"] = f"{float(handler_time):.2f}"
suite["build_time"] = f"{float(instance.build_time):.2f}"
testcases = [] testcases = []

66
scripts/pylib/twister/twisterlib/runner.py
View file

@ -263,27 +263,31 @@ class CMake:
if self.cwd: if self.cwd:
kwargs['cwd'] = self.cwd kwargs['cwd'] = self.cwd
start_time = time.time()
if sys.platform == 'linux': if sys.platform == 'linux':
p = self.jobserver.popen(cmd, **kwargs) p = self.jobserver.popen(cmd, **kwargs)
else: else:
p = subprocess.Popen(cmd, **kwargs) p = subprocess.Popen(cmd, **kwargs)
logger.debug(f'Running {"".join(cmd)}')
out, _ = p.communicate() out, _ = p.communicate()
results = {} ret = {}
duration = time.time() - start_time
self.instance.build_time += duration
if p.returncode == 0: if p.returncode == 0:
msg = "Finished building %s for %s" % (self.source_dir, self.platform.name) msg = f"Finished building {self.source_dir} for {self.platform.name} in {duration:.2f} seconds"
logger.debug(msg)
self.instance.status = "passed" self.instance.status = "passed"
if not self.instance.run: if not self.instance.run:
self.instance.add_missing_case_status("skipped", "Test was built only") self.instance.add_missing_case_status("skipped", "Test was built only")
results = {'msg': msg, "returncode": p.returncode, "instance": self.instance} ret = {"returncode": p.returncode}
if out: if out:
log_msg = out.decode(self.default_encoding) log_msg = out.decode(self.default_encoding)
with open(os.path.join(self.build_dir, self.log), "a", encoding=self.default_encoding) as log: with open(os.path.join(self.build_dir, self.log), "a", encoding=self.default_encoding) as log:
log.write(log_msg) log.write(log_msg)
else: else:
return None return None
else: else:
@ -310,12 +314,11 @@ class CMake:
self.instance.status = "error" self.instance.status = "error"
self.instance.reason = "Build failure" self.instance.reason = "Build failure"
results = { ret = {
"returncode": p.returncode, "returncode": p.returncode
"instance": self.instance,
} }
return results return ret
def run_cmake(self, args="", filter_stages=[]): def run_cmake(self, args="", filter_stages=[]):
@ -384,18 +387,24 @@ class CMake:
if self.cwd: if self.cwd:
kwargs['cwd'] = self.cwd kwargs['cwd'] = self.cwd
start_time = time.time()
if sys.platform == 'linux': if sys.platform == 'linux':
p = self.jobserver.popen(cmd, **kwargs) p = self.jobserver.popen(cmd, **kwargs)
else: else:
p = subprocess.Popen(cmd, **kwargs) p = subprocess.Popen(cmd, **kwargs)
out, _ = p.communicate() out, _ = p.communicate()
duration = time.time() - start_time
self.instance.build_time += duration
if p.returncode == 0: if p.returncode == 0:
filter_results = self.parse_generated(filter_stages) filter_results = self.parse_generated(filter_stages)
msg = "Finished building %s for %s" % (self.source_dir, self.platform.name) msg = f"Finished running cmake {self.source_dir} for {self.platform.name} in {duration:.2f} seconds"
logger.debug(msg) logger.debug(msg)
results = {'msg': msg, 'filter': filter_results} ret = {
'returncode': p.returncode,
'filter': filter_results
}
else: else:
self.instance.status = "error" self.instance.status = "error"
self.instance.reason = "Cmake build failure" self.instance.reason = "Cmake build failure"
@ -404,7 +413,7 @@ class CMake:
tc.status = self.instance.status tc.status = self.instance.status
logger.error("Cmake build failure: %s for %s" % (self.source_dir, self.platform.name)) logger.error("Cmake build failure: %s for %s" % (self.source_dir, self.platform.name))
results = {"returncode": p.returncode} ret = {"returncode": p.returncode}
if out: if out:
os.makedirs(self.build_dir, exist_ok=True) os.makedirs(self.build_dir, exist_ok=True)
@ -412,7 +421,7 @@ class CMake:
log_msg = out.decode(self.default_encoding) log_msg = out.decode(self.default_encoding)
log.write(log_msg) log.write(log_msg)
return results return ret
class FilterBuilder(CMake): class FilterBuilder(CMake):
@ -502,14 +511,14 @@ class FilterBuilder(CMake):
edt = pickle.load(f) edt = pickle.load(f)
else: else:
edt = None edt = None
res = expr_parser.parse(self.testsuite.filter, filter_data, edt) ret = expr_parser.parse(self.testsuite.filter, filter_data, edt)
except (ValueError, SyntaxError) as se: except (ValueError, SyntaxError) as se:
sys.stderr.write( sys.stderr.write(
"Failed processing %s\n" % self.testsuite.yamlfile) "Failed processing %s\n" % self.testsuite.yamlfile)
raise se raise se
if not res: if not ret:
return {os.path.join(self.platform.name, self.testsuite.name): True} return {os.path.join(self.platform.name, self.testsuite.name): True}
else: else:
return {os.path.join(self.platform.name, self.testsuite.name): False} return {os.path.join(self.platform.name, self.testsuite.name): False}
@ -584,12 +593,12 @@ class ProjectBuilder(FilterBuilder):
self.instance.setup_handler(self.env) self.instance.setup_handler(self.env)
if op == "filter": if op == "filter":
res = self.cmake(filter_stages=self.instance.filter_stages) ret = self.cmake(filter_stages=self.instance.filter_stages)
if self.instance.status in ["failed", "error"]: if self.instance.status in ["failed", "error"]:
pipeline.put({"op": "report", "test": self.instance}) pipeline.put({"op": "report", "test": self.instance})
else: else:
# Here we check the dt/kconfig filter results coming from running cmake # Here we check the dt/kconfig filter results coming from running cmake
if self.instance.name in res['filter'] and res['filter'][self.instance.name]: if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name) logger.debug("filtering %s" % self.instance.name)
self.instance.status = "filtered" self.instance.status = "filtered"
self.instance.reason = "runtime filter" self.instance.reason = "runtime filter"
@ -600,8 +609,8 @@ class ProjectBuilder(FilterBuilder):
pipeline.put({"op": "cmake", "test": self.instance}) pipeline.put({"op": "cmake", "test": self.instance})
# The build process, call cmake and build with configured generator # The build process, call cmake and build with configured generator
if op == "cmake": elif op == "cmake":
res = self.cmake() ret = self.cmake()
if self.instance.status in ["failed", "error"]: if self.instance.status in ["failed", "error"]:
pipeline.put({"op": "report", "test": self.instance}) pipeline.put({"op": "report", "test": self.instance})
elif self.options.cmake_only: elif self.options.cmake_only:
@ -610,7 +619,7 @@ class ProjectBuilder(FilterBuilder):
pipeline.put({"op": "report", "test": self.instance}) pipeline.put({"op": "report", "test": self.instance})
else: else:
# Here we check the runtime filter results coming from running cmake # Here we check the runtime filter results coming from running cmake
if self.instance.name in res['filter'] and res['filter'][self.instance.name]: if self.instance.name in ret['filter'] and ret['filter'][self.instance.name]:
logger.debug("filtering %s" % self.instance.name) logger.debug("filtering %s" % self.instance.name)
self.instance.status = "filtered" self.instance.status = "filtered"
self.instance.reason = "runtime filter" self.instance.reason = "runtime filter"
@ -622,8 +631,8 @@ class ProjectBuilder(FilterBuilder):
elif op == "build": elif op == "build":
logger.debug("build test: %s" % self.instance.name) logger.debug("build test: %s" % self.instance.name)
res = self.build() ret = self.build()
if not res: if not ret:
self.instance.status = "error" self.instance.status = "error"
self.instance.reason = "Build Failure" self.instance.reason = "Build Failure"
pipeline.put({"op": "report", "test": self.instance}) pipeline.put({"op": "report", "test": self.instance})
@ -634,7 +643,7 @@ class ProjectBuilder(FilterBuilder):
results.skipped_runtime += 1 results.skipped_runtime += 1
self.instance.add_missing_case_status("skipped", self.instance.reason) self.instance.add_missing_case_status("skipped", self.instance.reason)
if res.get('returncode', 1) > 0: if ret.get('returncode', 1) > 0:
self.instance.add_missing_case_status("blocked", self.instance.reason) self.instance.add_missing_case_status("blocked", self.instance.reason)
pipeline.put({"op": "report", "test": self.instance}) pipeline.put({"op": "report", "test": self.instance})
else: else:
@ -1040,13 +1049,10 @@ class ProjectBuilder(FilterBuilder):
self.options.extra_args, # CMake extra args self.options.extra_args, # CMake extra args
self.instance.build_dir, self.instance.build_dir,
) )
return self.run_cmake(args,filter_stages)
res = self.run_cmake(args,filter_stages)
return res
def build(self): def build(self):
res = self.run_build(['--build', self.build_dir]) return self.run_build(['--build', self.build_dir])
return res
def run(self): def run(self):
@ -1289,11 +1295,11 @@ class TwisterRunner:
processes = [] processes = []
for job in range(self.jobs): for _ in range(self.jobs):
logger.debug(f"Launch process {job}")
p = Process(target=self.pipeline_mgr, args=(pipeline, done, lock, self.results, )) p = Process(target=self.pipeline_mgr, args=(pipeline, done, lock, self.results, ))
processes.append(p) processes.append(p)
p.start() p.start()
logger.debug(f"Launched {self.jobs} jobs")
try: try:
for p in processes: for p in processes:

1
scripts/pylib/twister/twisterlib/testinstance.py
View file

@ -50,6 +50,7 @@ class TestInstance:
self.handler = None self.handler = None
self.outdir = outdir self.outdir = outdir
self.execution_time = 0 self.execution_time = 0
self.build_time = 0
self.retries = 0 self.retries = 0
self.name = os.path.join(platform.name, testsuite.name) self.name = os.path.join(platform.name, testsuite.name)

1
scripts/pylib/twister/twisterlib/testplan.py
View file

@ -416,7 +416,6 @@ class TestPlan:
self.platforms.append(platform) self.platforms.append(platform)
if not platform_config.get('override_default_platforms', False): if not platform_config.get('override_default_platforms', False):
if platform.default: if platform.default:
logger.debug(f"adding {platform.name} to default platforms")
self.default_platforms.append(platform.name) self.default_platforms.append(platform.name)
else: else:
if platform.name in platform_config.get('default_platforms', []): if platform.name in platform_config.get('default_platforms', []):