From 00bc6279701ddea0377ac899a40d6c11f24347d5 Mon Sep 17 00:00:00 2001 From: Abul Date: Fri, 5 Jun 2026 18:13:18 +0530 Subject: [PATCH 1/3] Implement Perfrunner initial --- .github/scripts/perf_comment.py | 88 +++++++++++++++++++++ .github/workflows/perf.yml | 93 ++++++++++++++++++++++ tools/perfrunner/.gitignore | 7 ++ tools/perfrunner/dub.json | 7 ++ tools/perfrunner/source/app.d | 61 +++++++++++++++ tools/perfrunner/source/cachegrind.d | 38 +++++++++ tools/perfrunner/source/metrics.d | 94 +++++++++++++++++++++++ tools/perfrunner/source/report.d | 75 ++++++++++++++++++ tools/perfrunner/source/runner.d | 17 ++++ tools/perfrunner/source/stats.d | 17 ++++ tools/perfrunner/source/workloads/hello.d | 6 ++ 11 files changed, 503 insertions(+) create mode 100644 .github/scripts/perf_comment.py create mode 100644 .github/workflows/perf.yml create mode 100644 tools/perfrunner/.gitignore create mode 100644 tools/perfrunner/dub.json create mode 100644 tools/perfrunner/source/app.d create mode 100644 tools/perfrunner/source/cachegrind.d create mode 100644 tools/perfrunner/source/metrics.d create mode 100644 tools/perfrunner/source/report.d create mode 100644 tools/perfrunner/source/runner.d create mode 100644 tools/perfrunner/source/stats.d create mode 100644 tools/perfrunner/source/workloads/hello.d diff --git a/.github/scripts/perf_comment.py b/.github/scripts/perf_comment.py new file mode 100644 index 000000000000..abcabd2adb44 --- /dev/null +++ b/.github/scripts/perf_comment.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Render the perfrunner results.json into a sticky PR comment. + +also upserts a single comment identified by a hidden marker so +pushes update one comment instead of spamming. +""" + +import json +import os +import sys +import urllib.request + +MARKER = "" + + +def fmt_value(value, unit): + if unit == "count": + return f"{value / 1e6:,.1f} M" + if unit == "bytes": + return f"{value / (1024 * 1024):.2f} MB" + if unit == "kb": + return f"{value / 1024:.0f} MB" + return str(value) + + +def fmt_delta(pct): + return f"+{pct:.2f}%" if pct > 0 else f"{pct:.2f}%" + + +def render(results): + lines = [ + MARKER, + "### DMD perf check", + "", + "| Metric | Base | PR | delta |", + "|--------|------|----|-------|", + ] + for m in results["metrics"]: + lines.append("| {} | {} | {} | {} |".format( + m["label"], + fmt_value(m["base"], m["unit"]), + fmt_value(m["head"], m["unit"]), + fmt_delta(m["delta_pct"]), + )) + return "\n".join(lines) + "\n" + + +def api(method, url, token, payload=None): + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Accept", "application/vnd.github+json") + if data: + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read() or "null") + + +def upsert(body, repo, pr, token): + base = f"https://api.github.com/repos/{repo}" + comments = api("GET", f"{base}/issues/{pr}/comments?per_page=100", token) + existing = next((c for c in comments if MARKER in (c.get("body") or "")), None) + if existing: + api("PATCH", f"{base}/issues/comments/{existing['id']}", token, {"body": body}) + else: + api("POST", f"{base}/issues/{pr}/comments", token, {"body": body}) + + +def main(): + if len(sys.argv) != 2: + sys.exit("usage: perf_comment.py results.json") + + with open(sys.argv[1]) as f: + results = json.load(f) + + body = render(results) + print(body) + + token = os.environ.get("GITHUB_TOKEN") + repo = os.environ.get("REPO") + pr = os.environ.get("PR_NUMBER") + if token and repo and pr: + upsert(body, repo, pr, token) + print(f"upserted comment on {repo}#{pr}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 000000000000..a1e4bcfab2cc --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,93 @@ +name: perf + +on: + pull_request: + paths-ignore: + - 'spec/**' + - 'changelog/**' + - '**/*.md' + push: + branches: [master] + +concurrency: + group: perf-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + perf: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + OS_NAME: linux + MODEL: 64 + FULL_BUILD: false + HOST_DMD: dmd-2.112.0 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set parallelism + run: echo "N=$(nproc)" >> "$GITHUB_ENV" + + - name: Compute base/head SHAs + id: refs + run: | + set -uexo pipefail + HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" + git fetch --no-tags origin master + BASE_SHA="$(git merge-base "$HEAD_SHA" origin/master)" + echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT" + echo "base=$BASE_SHA" >> "$GITHUB_OUTPUT" + echo "branch=${GITHUB_BASE_REF:-master}" >> "$GITHUB_OUTPUT" + + - name: Install prerequisites + run: | + sudo apt-get update + sudo apt-get install -y valgrind time + valgrind --version + + - name: Install host compiler + run: ci/run.sh install_host_compiler + + - name: Build dmd at base and head + run: | + set -uexo pipefail + build_ref() { + local dir="$RUNNER_TEMP/$2/dmd" + mkdir -p "$RUNNER_TEMP/$2" + git worktree add --force "$dir" "$1" + ( cd "$dir" && ci/run.sh setup_repos "${{ steps.refs.outputs.branch }}" && ci/run.sh build 0 ) + } + build_ref "${{ steps.refs.outputs.base }}" base + build_ref "${{ steps.refs.outputs.head }}" head + + - name: Measure + run: | + set -uexo pipefail + source ~/dlang/*/activate + built=generated/$OS_NAME/release/$MODEL/dmd + cd tools/perfrunner + dub run -- \ + --base-dmd "$RUNNER_TEMP/base/dmd/$built" \ + --head-dmd "$RUNNER_TEMP/head/dmd/$built" \ + --base-sha "${{ steps.refs.outputs.base }}" \ + --head-sha "${{ steps.refs.outputs.head }}" \ + --pr "${{ github.event.pull_request.number || 0 }}" \ + --host-dmd "${HOST_DMD#dmd-}" \ + --out "$GITHUB_WORKSPACE/results.json" + + - name: Post sticky PR comment + if: github.event_name == 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: python3 .github/scripts/perf_comment.py results.json diff --git a/tools/perfrunner/.gitignore b/tools/perfrunner/.gitignore new file mode 100644 index 000000000000..f89d9b4adda1 --- /dev/null +++ b/tools/perfrunner/.gitignore @@ -0,0 +1,7 @@ +.dub/ +perfrunner +perfrunner.exe +*-test-* +*.o +*.obj +results.json diff --git a/tools/perfrunner/dub.json b/tools/perfrunner/dub.json new file mode 100644 index 000000000000..dc169bd7eaa9 --- /dev/null +++ b/tools/perfrunner/dub.json @@ -0,0 +1,7 @@ +{ + "name": "perfrunner", + "description": "DMD performance measurement (dev-phase)", + "targetType": "executable", + "sourcePaths": ["source"], + "excludedSourceFiles": ["source/workloads/*"] +} diff --git a/tools/perfrunner/source/app.d b/tools/perfrunner/source/app.d new file mode 100644 index 000000000000..3324c5b341aa --- /dev/null +++ b/tools/perfrunner/source/app.d @@ -0,0 +1,61 @@ +module app; + +import std.file : mkdirRecurse, tempDir, write; +import std.getopt : getopt; +import std.path : buildPath, dirName; +import std.stdio : stderr, writeln; + +import metrics : measure, initials; +import report : MetricResult, render, Report; + +// Initial workload: the one source file compile to measure DMD. +enum workload = buildPath(__FILE_FULL_PATH__.dirName, "workloads", "hello.d"); + +version (unittest) {} else +int main(string[] args) +{ + string baseDmd, headDmd, baseSha, headSha, hostDmd; + string os = "ubuntu-latest"; + string outPath = "results.json"; + long pr; + + auto help = getopt(args, + "base-dmd", "path to the base (merge-base) dmd binary", &baseDmd, + "head-dmd", "path to the head (PR) dmd binary", &headDmd, + "base-sha", "base commit sha (metadata)", &baseSha, + "head-sha", "head commit sha (metadata)", &headSha, + "pr", "pull request number (metadata)", &pr, + "os", "runner OS label (metadata)", &os, + "host-dmd", "bootstrap dmd version (metadata)", &hostDmd, + "out", "where to write results.json", &outPath, + ); + + if (help.helpWanted) + { + writeln("usage: perfrunner --base-dmd --head-dmd " + ~ "[--base-sha --head-sha --pr ] --out results.json"); + return 0; + } + + if (baseDmd.length == 0 || headDmd.length == 0) + { + stderr.writeln("error: --base-dmd and --head-dmd are required"); + return 2; + } + + auto tmp = buildPath(tempDir, "perfrunner"); + mkdirRecurse(tmp); + + auto base = measure(baseDmd, workload, tmp, "base"); + auto head = measure(headDmd, workload, tmp, "head"); + + MetricResult[] metrics; + foreach (def; initials) + metrics ~= MetricResult(def.id, def.label, def.unit, def.method, + base[def.id], head[def.id]); + + auto rep = Report(baseSha, "merge-base", headSha, pr, os, hostDmd, metrics); + write(outPath, render(rep)); + writeln("wrote ", outPath); + return 0; +} diff --git a/tools/perfrunner/source/cachegrind.d b/tools/perfrunner/source/cachegrind.d new file mode 100644 index 000000000000..7b30adc7372f --- /dev/null +++ b/tools/perfrunner/source/cachegrind.d @@ -0,0 +1,38 @@ +module cachegrind; + +import std.array : replace; +import std.conv : to; +import std.path : buildPath; +import std.regex : ctRegex, matchFirst; + +import runner : run; + +private enum iRefsRe = ctRegex!(`I\s+refs:\s+([\d,]+)`); + +// Parse the "I refs:" instruction count out of cachegrind's output. +long parseIRefs(string output) +{ + auto m = matchFirst(output, iRefsRe); + if (m.empty) + throw new Exception("could not parse cachegrind 'I refs:'"); + return m[1].replace(",", "").to!long; +} + +// Compile the workload under cachegrind +long instructions(string dmd, string[] dflags, string workload, string tmp, string tag) +{ + auto obj = buildPath(tmp, tag ~ ".o"); + auto cgOut = buildPath(tmp, tag ~ ".cgout"); + auto cmd = ["valgrind", "--tool=cachegrind", "--cachegrind-out-file=" ~ cgOut, + dmd, "-c"] ~ dflags ~ [workload, "-of=" ~ obj]; + auto r = run(cmd); + if (r.status != 0) + throw new Exception("cachegrind failed:\n" ~ r.output); + return parseIRefs(r.output); +} + +unittest +{ + auto sample = "==42== I refs: 1,234,500,000\n"; + assert(parseIRefs(sample) == 1_234_500_000); +} diff --git a/tools/perfrunner/source/metrics.d b/tools/perfrunner/source/metrics.d new file mode 100644 index 000000000000..1145140d002a --- /dev/null +++ b/tools/perfrunner/source/metrics.d @@ -0,0 +1,94 @@ +module metrics; + +import std.conv : to; +import std.file : copy, exists, getSize, remove; +import std.path : buildPath; +import std.regex : ctRegex, matchFirst; + +import cachegrind : instructions; +import runner : run; + +struct MetricDef +{ + string id; + string label; + string unit; + string method; +} + +// Some initial metrics to measure will add more later +immutable MetricDef[] initials = [ + MetricDef("compile_hello_debug_instr", "compile hello.d (instr)", "count", "cachegrind"), + MetricDef("compile_hello_release_instr", "compile hello.d -O (instr)", "count", "cachegrind"), + MetricDef("dmd_binary_size", "dmd binary size (stripped)", "bytes", "stat"), + MetricDef("hello_binary_size", "hello binary size", "bytes", "stat"), + MetricDef("hello_max_rss", "peak RSS (compile hello.d)", "kb", "time -v"), +]; + +// Measure every metric for one dmd binary. `tag` ("base"/"head") +// keeps the two runs' temp files apart +long[string] measure(string dmd, string workload, string tmp, string tag) +{ + return [ + "compile_hello_debug_instr": instructions(dmd, [], workload, tmp, tag ~ "-dbg"), + "compile_hello_release_instr": instructions(dmd, ["-O", "-release"], workload, tmp, tag ~ "-rel"), + "dmd_binary_size": strippedSize(dmd, buildPath(tmp, tag ~ "-dmd")), + "hello_binary_size": helloSize(dmd, workload, tmp, tag), + "hello_max_rss": maxRss(dmd, workload, tmp, tag), + ]; +} + +// Byte size of `binary` +private long strippedSize(string binary, string copyPath) +{ + if (exists(copyPath)) + remove(copyPath); + copy(binary, copyPath); + strip(copyPath); + return getSize(copyPath); +} + +// Compile the workload to an executable and its size in bytes +private long helloSize(string dmd, string workload, string tmp, string tag) +{ + auto exe = buildPath(tmp, tag ~ "-hello"); + auto r = run([dmd, workload, "-of=" ~ exe]); + if (r.status != 0) + throw new Exception("compiling hello executable failed:\n" ~ r.output); + strip(exe); + return getSize(exe); +} + +private void strip(string path) +{ + auto r = run(["strip", path]); + if (r.status != 0) + throw new Exception("strip failed:\n" ~ r.output); +} + +// Peak RSS (KiB) of compiling the workload (/usr/bin/time) +private long maxRss(string dmd, string workload, string tmp, string tag) +{ + auto obj = buildPath(tmp, tag ~ "-rss.o"); + auto r = run(["/usr/bin/time", "-v", dmd, "-c", workload, "-of=" ~ obj]); + if (r.status != 0) + throw new Exception("/usr/bin/time failed:\n" ~ r.output); + return parseMaxRss(r.output); +} + +private enum rssRe = ctRegex!(`Maximum resident set size \(kbytes\):\s+(\d+)`); + +// Pull the max-RSS value (KiB) out of `/usr/bin/time -v` output. +long parseMaxRss(string output) +{ + auto m = matchFirst(output, rssRe); + if (m.empty) + throw new Exception("could not parse max RSS"); + return m[1].to!long; +} + +unittest +{ + auto sample = "\tMaximum resident set size (kbytes): 184320\n"; + assert(parseMaxRss(sample) == 184320); +} diff --git a/tools/perfrunner/source/report.d b/tools/perfrunner/source/report.d new file mode 100644 index 000000000000..a28e7217a086 --- /dev/null +++ b/tools/perfrunner/source/report.d @@ -0,0 +1,75 @@ +module report; + +import std.json : JSONValue, parseJSON; +import std.math : round; + +import stats : deltaPct; + +struct MetricResult +{ + string id; + string label; + string unit; + string method; + long base; + long head; +} + +struct Report +{ + string baseSha; + string baseRef; + string headSha; + long pr; + string os; + string hostDmd; + MetricResult[] metrics; +} + +// Serialise a report to the initial schema +string render(Report rep) +{ + JSONValue[] metrics; + foreach (m; rep.metrics) + { + metrics ~= JSONValue([ + "id": JSONValue(m.id), + "label": JSONValue(m.label), + "unit": JSONValue(m.unit), + "method": JSONValue(m.method), + "base": JSONValue(m.base), + "head": JSONValue(m.head), + "delta_pct": JSONValue(round(deltaPct(m.base, m.head) * 100) / 100.0), + ]); + } + + JSONValue root = [ + "schema_version": JSONValue(1), + "base": JSONValue(["sha": JSONValue(rep.baseSha), "ref": JSONValue(rep.baseRef)]), + "head": JSONValue(["sha": JSONValue(rep.headSha), "pr": JSONValue(rep.pr)]), + "runner": JSONValue(["os": JSONValue(rep.os), "host_dmd": JSONValue(rep.hostDmd)]), + "metrics": JSONValue(metrics), + ]; + + return root.toPrettyString(); +} + +unittest +{ + auto rep = Report("base1", "merge-base", "head1", 7, "ubuntu-latest", "2.112.0", + [MetricResult("compile_hello_debug_instr", "compile hello.d (instr)", + "count", "cachegrind", 1000, 1010)]); + + auto j = parseJSON(render(rep)); + assert(j["schema_version"].integer == 1); + assert(j["base"]["sha"].str == "base1"); + assert(j["head"]["pr"].integer == 7); + assert(j["metrics"].array.length == 1); + + auto m = j["metrics"][0]; + assert(m["id"].str == "compile_hello_debug_instr"); + assert(m["base"].integer == 1000); + + import std.math : isClose; + assert(isClose(m["delta_pct"].floating, 1.0)); +} diff --git a/tools/perfrunner/source/runner.d b/tools/perfrunner/source/runner.d new file mode 100644 index 000000000000..95efd815e8fe --- /dev/null +++ b/tools/perfrunner/source/runner.d @@ -0,0 +1,17 @@ +module runner; + +import std.process : execute; + +// Outcome +struct RunResult +{ + int status; // process exit code + string output; // combined stdout + stderr +} + +// Run `cmd` and capture its exit code and output +RunResult run(string[] cmd) +{ + auto r = execute(cmd); + return RunResult(r.status, r.output); +} diff --git a/tools/perfrunner/source/stats.d b/tools/perfrunner/source/stats.d new file mode 100644 index 000000000000..0e13f59c06dd --- /dev/null +++ b/tools/perfrunner/source/stats.d @@ -0,0 +1,17 @@ +module stats; + +// Percent change from `base` to `head`. +double deltaPct(double base, double head) +{ + if (base == 0) + return 0; + return (head - base) / base * 100.0; +} + +unittest +{ + import std.math : isClose; + assert(isClose(deltaPct(100, 101), 1.0)); + assert(isClose(deltaPct(200, 150), -25.0)); + assert(deltaPct(0, 5) == 0); +} diff --git a/tools/perfrunner/source/workloads/hello.d b/tools/perfrunner/source/workloads/hello.d new file mode 100644 index 000000000000..a708e640b550 --- /dev/null +++ b/tools/perfrunner/source/workloads/hello.d @@ -0,0 +1,6 @@ +import std.stdio; + +void main() +{ + writeln("hello"); +} From b92d2b8a4a6a393a51ba9886e3f5de1d96e402ae Mon Sep 17 00:00:00 2001 From: Abul Date: Fri, 5 Jun 2026 18:26:42 +0530 Subject: [PATCH 2/3] Disable other workflows as of now --- .github/workflows/{bootstrap.yml => bootstrap.yml.disabled} | 0 .../{build_with_dub.yml => build_with_dub.yml.disabled} | 0 .github/workflows/{main.yml => main.yml.disabled} | 0 .github/workflows/{nightlies.yml => nightlies.yml.disabled} | 0 .github/workflows/{release.yml => release.yml.disabled} | 0 .../workflows/{runnable_cxx.yml => runnable_cxx.yml.disabled} | 0 tools/perfrunner/source/metrics.d | 2 +- tools/perfrunner/source/runner.d | 2 +- 8 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{bootstrap.yml => bootstrap.yml.disabled} (100%) rename .github/workflows/{build_with_dub.yml => build_with_dub.yml.disabled} (100%) rename .github/workflows/{main.yml => main.yml.disabled} (100%) rename .github/workflows/{nightlies.yml => nightlies.yml.disabled} (100%) rename .github/workflows/{release.yml => release.yml.disabled} (100%) rename .github/workflows/{runnable_cxx.yml => runnable_cxx.yml.disabled} (100%) diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml.disabled similarity index 100% rename from .github/workflows/bootstrap.yml rename to .github/workflows/bootstrap.yml.disabled diff --git a/.github/workflows/build_with_dub.yml b/.github/workflows/build_with_dub.yml.disabled similarity index 100% rename from .github/workflows/build_with_dub.yml rename to .github/workflows/build_with_dub.yml.disabled diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml.disabled similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/main.yml.disabled diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml.disabled similarity index 100% rename from .github/workflows/nightlies.yml rename to .github/workflows/nightlies.yml.disabled diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml.disabled similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/release.yml.disabled diff --git a/.github/workflows/runnable_cxx.yml b/.github/workflows/runnable_cxx.yml.disabled similarity index 100% rename from .github/workflows/runnable_cxx.yml rename to .github/workflows/runnable_cxx.yml.disabled diff --git a/tools/perfrunner/source/metrics.d b/tools/perfrunner/source/metrics.d index 1145140d002a..deb5afdb3ce4 100644 --- a/tools/perfrunner/source/metrics.d +++ b/tools/perfrunner/source/metrics.d @@ -16,7 +16,7 @@ struct MetricDef string method; } -// Some initial metrics to measure will add more later +// Some initial metrics to measure will add more later immutable MetricDef[] initials = [ MetricDef("compile_hello_debug_instr", "compile hello.d (instr)", "count", "cachegrind"), MetricDef("compile_hello_release_instr", "compile hello.d -O (instr)", "count", "cachegrind"), diff --git a/tools/perfrunner/source/runner.d b/tools/perfrunner/source/runner.d index 95efd815e8fe..fb3ec8257aef 100644 --- a/tools/perfrunner/source/runner.d +++ b/tools/perfrunner/source/runner.d @@ -2,7 +2,7 @@ module runner; import std.process : execute; -// Outcome +// Outcome struct RunResult { int status; // process exit code From c779a35ee7e252abbddd6120b5eb28785700e1f7 Mon Sep 17 00:00:00 2001 From: Abul Date: Fri, 5 Jun 2026 18:45:16 +0530 Subject: [PATCH 3/3] Add deliberate busy loop --- compiler/src/dmd/main.d | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compiler/src/dmd/main.d b/compiler/src/dmd/main.d index 28163b9dfbd7..26b376a14abb 100644 --- a/compiler/src/dmd/main.d +++ b/compiler/src/dmd/main.d @@ -629,6 +629,15 @@ private int tryMain(const(char)[][] argv, out Param params) backend_init(params, driverParams, target); + // deliberate busy loop to see if perf bot picks it up + { + long dummy = 0; + foreach (i; 0 .. 5_000_000) + dummy += i * 3; + if (dummy == -1) + fputs("", stderr); + } + // Do semantic analysis foreach (m; modules) {