Skip to content

Commit e2612f7

Browse files
apaolilloEstAK
apaolillo
authored and
EstAK
committed
perf: Add dd bench and harden flamegraph checks (open-s4c#324)
Introduce a small DD benchmark that burns CPU by streaming /dev/urandom into /dev/null, giving a reliable workload for perf sampling and flamegraph generation (unlike sleep). Make PerfReportWrap skip flamegraph generation when perf produces no folded stack counts, emitting a warning instead of failing. Refactor the flamegraph test into a suite with two sub-campaigns: sleep (no samples leads to graceful skip) and dd (real samples generates a SVG flamegraph), and add a dedicated fzf-based campaign for interactively browsing dd runs. Signed-off-by: Antonio Paolillo <apaolill@gmail.com>
1 parent bcddecd commit e2612f7

4 files changed

Lines changed: 170 additions & 47 deletions

File tree

‎benchkit/benches/small/dd.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright (C) 2026 Vrije Universiteit Brussel. All rights reserved.
2+
# SPDX-License-Identifier: MIT
3+
4+
from benchkit.core.bktypes.callresults import RunResult
5+
from benchkit.core.bktypes.contexts import RunContext
6+
7+
8+
class DDBench:
9+
"""
10+
DD benchmark (benchkit core protocol).
11+
12+
CPU-intensive workload using dd to read from /dev/urandom and write to
13+
/dev/null. Useful for testing perf-based profiling (e.g. flamegraphs)
14+
because it generates measurable CPU activity, unlike sleep.
15+
16+
- run: execute dd if=/dev/urandom of=/dev/null bs=1M count=block_count
17+
"""
18+
19+
def run(
20+
self,
21+
ctx: RunContext,
22+
block_count: int,
23+
) -> RunResult:
24+
out = ctx.exec(
25+
argv=[
26+
"dd",
27+
"if=/dev/urandom",
28+
"of=/dev/null",
29+
"bs=1M",
30+
f"count={block_count}",
31+
],
32+
cwd=ctx.build_result.build_dir if ctx.build_result is not None else None,
33+
print_output=False,
34+
ignore_ret_codes=(1,),
35+
)
36+
return RunResult(outputs=[out])

‎benchkit/commandwrappers/perf.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,14 @@ def post_run_hook_flamegraph(
876876
)
877877
perf_folded_pathname.write_text(out_folded.strip())
878878

879+
if not out_folded.strip():
880+
print(
881+
"[WARNING] No perf stack counts found. "
882+
"Skipping flamegraph generation for this run.",
883+
file=sys.stderr,
884+
)
885+
return
886+
879887
flamegraph_command = self._flamegraph_command(
880888
title=flamegraph_title,
881889
subtitle=flamegraph_subtitle,

‎tests/campaigns/campaign_flame.py‎

Lines changed: 68 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,31 @@
22
# Copyright (C) 2024 Vrije Universiteit Brussel. All rights reserved.
33
# SPDX-License-Identifier: MIT
44

5-
from benchmarks.sleep import SleepBench
5+
"""
6+
Flamegraph test campaign.
67
7-
from benchkit.campaign import CampaignIterateVariables
8-
from benchkit.commandwrappers.perf import PerfReportWrap, enable_non_sudo_perf
9-
from benchkit.platforms import get_current_platform
10-
from benchkit.utils.dir import caller_dir
11-
from benchkit.utils.git import clone_repo
8+
Two sub-campaigns exercise perf-based flame graph generation:
129
10+
1. **sleep** — a workload that produces zero CPU samples.
11+
Validates the graceful no-samples path (warning instead of crash).
1312
14-
def main() -> None:
15-
platform = get_current_platform()
16-
enable_non_sudo_perf(comm_layer=platform.comm)
13+
2. **dd** — a CPU-intensive workload (dd if=/dev/urandom of=/dev/null).
14+
Validates actual flamegraph SVG generation with real perf data.
15+
"""
1716

18-
flamegraph_path = caller_dir() / "deps/FlameGraph"
19-
clone_repo(
20-
repo_url="https://github.com/brendangregg/FlameGraph.git",
21-
repo_src_dir=flamegraph_path,
22-
commit="cd9ee4c4449775a2f867acf31c84b7fe4b132ad5",
23-
)
17+
from benchkit import CampaignCartesianProduct
18+
from benchkit.benches.small.dd import DDBench
19+
from benchkit.benches.small.sleep import SleepBench
20+
from benchkit.campaign import CampaignSuite
21+
from benchkit.commandwrappers.perf import PerfReportWrap, enable_non_sudo_perf
22+
from benchkit.platforms import get_current_platform
23+
from benchkit.utils.dir import get_tools_dir
2424

25-
perf_wrapper = PerfReportWrap(
26-
freq=99,
27-
# freq=10,
28-
report_interactive=False,
29-
report_file=True,
30-
flamegraph_path=flamegraph_path,
31-
)
3225

33-
def flame_post_hook(
26+
def _make_flame_post_hook(perf_wrapper):
27+
"""Return a post-run hook that generates a flame graph for each run."""
28+
29+
def hook(
3430
experiment_results_lines,
3531
record_data_dir,
3632
write_record_file_fun,
@@ -43,35 +39,60 @@ def flame_post_hook(
4339
flamegraph_fontsize=14,
4440
)
4541

46-
campaign = CampaignIterateVariables(
47-
name="flame",
48-
benchmark=SleepBench(
49-
command_wrappers=[perf_wrapper],
50-
post_run_hooks=[
51-
perf_wrapper.post_run_hook_report,
52-
flame_post_hook,
53-
],
54-
),
42+
return hook
43+
44+
45+
def main() -> None:
46+
platform = get_current_platform()
47+
enable_non_sudo_perf(comm_layer=platform.comm)
48+
49+
flamegraph_dir = get_tools_dir(None) / "FlameGraph"
50+
51+
# --- shared perf wrapper (one instance is fine for sequential campaigns) ---
52+
perf_wrapper = PerfReportWrap(
53+
freq=99,
54+
report_interactive=False,
55+
report_file=True,
56+
flamegraph_path=flamegraph_dir,
57+
)
58+
perf_wrapper.fetch_flamegraph()
59+
60+
flame_hook = _make_flame_post_hook(perf_wrapper)
61+
62+
# --- Campaign 1: sleep (no CPU samples → graceful skip) ---
63+
campaign_sleep = CampaignCartesianProduct(
64+
name="flame_sleep",
65+
benchmark=SleepBench(),
66+
variables={
67+
"duration_seconds": [1],
68+
},
5569
nb_runs=1,
56-
variables=[
57-
{
58-
"duration_seconds": 1,
59-
},
60-
{
61-
"duration_seconds": 2,
62-
},
70+
command_wrappers=[perf_wrapper],
71+
post_run_hooks=[
72+
perf_wrapper.post_run_hook_report,
73+
flame_hook,
6374
],
64-
constants=None,
65-
debug=False,
66-
gdb=False,
67-
enable_data_dir=True,
75+
platform=platform,
6876
)
6977

70-
campaign.run()
78+
# --- Campaign 2: dd (CPU-intensive → real flamegraph) ---
79+
campaign_dd = CampaignCartesianProduct(
80+
name="flame_dd",
81+
benchmark=DDBench(),
82+
variables={
83+
"block_count": [50],
84+
},
85+
nb_runs=1,
86+
command_wrappers=[perf_wrapper],
87+
post_run_hooks=[
88+
perf_wrapper.post_run_hook_report,
89+
flame_hook,
90+
],
91+
platform=platform,
92+
)
7193

72-
results_path = campaign.base_data_dir()
73-
perf_wrapper.fzf_report(search_dir=results_path)
74-
perf_wrapper.fzf_flamegraph(search_dir=results_path)
94+
suite = CampaignSuite(campaigns=[campaign_sleep, campaign_dd])
95+
suite.run_suite()
7596

7697

7798
if __name__ == "__main__":

‎tests/campaigns/campaign_flame_fzf.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env python3
2+
# Copyright (C) 2024 Vrije Universiteit Brussel. All rights reserved.
3+
# SPDX-License-Identifier: MIT
4+
from benchkit import CampaignCartesianProduct
5+
from benchkit.benches.small.dd import DDBench
6+
from benchkit.commandwrappers.perf import PerfReportWrap, enable_non_sudo_perf
7+
from benchkit.platforms import get_current_platform
8+
from benchkit.utils.dir import get_tools_dir
9+
10+
11+
def main() -> None:
12+
platform = get_current_platform()
13+
enable_non_sudo_perf(comm_layer=platform.comm)
14+
15+
flamegraph_dir = get_tools_dir(None) / "FlameGraph"
16+
perf_wrapper = PerfReportWrap(
17+
freq=99,
18+
report_interactive=False,
19+
report_file=True,
20+
flamegraph_path=flamegraph_dir,
21+
)
22+
perf_wrapper.fetch_flamegraph()
23+
24+
def flame_post_hook(
25+
experiment_results_lines,
26+
record_data_dir,
27+
write_record_file_fun,
28+
):
29+
return perf_wrapper.post_run_hook_flamegraph(
30+
experiment_results_lines=experiment_results_lines,
31+
record_data_dir=record_data_dir,
32+
write_record_file_fun=write_record_file_fun,
33+
flamegraph_width=400,
34+
flamegraph_fontsize=14,
35+
)
36+
37+
campaign = CampaignCartesianProduct(
38+
name="flame_dd_fzf",
39+
benchmark=DDBench(),
40+
variables={"block_count": [50, 500]},
41+
nb_runs=1,
42+
command_wrappers=[perf_wrapper],
43+
post_run_hooks=[
44+
perf_wrapper.post_run_hook_report,
45+
flame_post_hook,
46+
],
47+
platform=platform,
48+
)
49+
50+
campaign.run()
51+
52+
results_path = campaign.base_data_dir()
53+
perf_wrapper.fzf_report(search_dir=results_path)
54+
perf_wrapper.fzf_flamegraph(search_dir=results_path)
55+
56+
57+
if __name__ == "__main__":
58+
main()

0 commit comments

Comments
 (0)