diff --git a/tests/benchmarks/latency_measure/boards/intel_adsp_ace15_mtpm_sim.conf b/tests/benchmarks/latency_measure/boards/intel_adsp_ace15_mtpm_sim.conf
new file mode 100644
index 00000000000..bad81b9ffbf
--- /dev/null
+++ b/tests/benchmarks/latency_measure/boards/intel_adsp_ace15_mtpm_sim.conf
@@ -0,0 +1,6 @@
+# Due to addition of busy threads running on other cores,
+# and the simulator runs in single thread bouncing through
+# all cores, we are wasting quite a bit of time just busy
+# waiting. This makes each simulator run too long for CI.
+# So limit CPU number to 1.
+CONFIG_MP_MAX_NUM_CPUS=1
diff --git a/tests/benchmarks/latency_measure/boards/intel_adsp_ace20_lnl_sim.conf b/tests/benchmarks/latency_measure/boards/intel_adsp_ace20_lnl_sim.conf
new file mode 100644
index 00000000000..bad81b9ffbf
--- /dev/null
+++ b/tests/benchmarks/latency_measure/boards/intel_adsp_ace20_lnl_sim.conf
@@ -0,0 +1,6 @@
+# Due to addition of busy threads running on other cores,
+# and the simulator runs in single thread bouncing through
+# all cores, we are wasting quite a bit of time just busy
+# waiting. This makes each simulator run too long for CI.
+# So limit CPU number to 1.
+CONFIG_MP_MAX_NUM_CPUS=1
diff --git a/tests/benchmarks/latency_measure/boards/intel_adsp_ace30_ptl_sim.conf b/tests/benchmarks/latency_measure/boards/intel_adsp_ace30_ptl_sim.conf
new file mode 100644
index 00000000000..bad81b9ffbf
--- /dev/null
+++ b/tests/benchmarks/latency_measure/boards/intel_adsp_ace30_ptl_sim.conf
@@ -0,0 +1,6 @@
+# Due to addition of busy threads running on other cores,
+# and the simulator runs in single thread bouncing through
+# all cores, we are wasting quite a bit of time just busy
+# waiting. This makes each simulator run too long for CI.
+# So limit CPU number to 1.
+CONFIG_MP_MAX_NUM_CPUS=1