From da5a310e34d6cbf6ff0b5dd758cf4ab56e6ebb94 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 17:32:30 +0100 Subject: [PATCH 01/29] Benchmarking automatically --- .github/workflows/benchmark.yml | 44 +++++++++++++++++++++++++++++++++ benchmark/Dockerfile | 37 +++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/Dockerfile diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..e8ccb5c67 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,44 @@ +name: snmalloc Benchmarking CI + +# The following should ensure that the workflow only runs a single set of actions +# for each PR. But it will not apply this to pushes to the main branch. +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ main ] + pull_request: + branches: [ main ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout + uses: actions/checkout@v3 + + # Builds the Docker image and runs the benchmarks + - name: Run Benchmarks + run: | + docker build -t snmalloc-bench -f benchmark/Dockerfile . --build-arg benchs=allt + + # Extracts the benchmark results from the Docker container + - name: Extract Benchmark Results + run: | + docker cp `docker run -d snmalloc-bench`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + + # Uploads the benchmark results as an artifact + - name: Upload Benchmark Results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: benchres.csv diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile new file mode 100644 index 000000000..0c72f068c --- /dev/null +++ b/benchmark/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 + +# Pull mimalloc-bench +RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates +RUN git clone https://github.com/daanx/mimalloc-bench &&\ + cd mimalloc-bench && \ + git reset --hard ffa530dbbe046532dfcb4a1b58ffc06e144aee60 + +WORKDIR /mimalloc-bench +# Install dependencies +RUN ./build-bench-env.sh packages + +# Build benchmarks +RUN ./build-bench-env.sh bench + +RUN ./build-bench-env.sh redis + +RUN ./build-bench-env.sh rocksdb + +RUN ./build-bench-env.sh lean + +RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt + +# Build allocator +ARG allocator=mi +RUN mkdir -p /snmalloc +COPY . /snmalloc + +RUN mkdir -p /snmalloc/build +WORKDIR /snmalloc/build +RUN cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +RUN ninja libsnmallocshim.so + +# Run benchmarks +ARG benchs=allt +WORKDIR /mimalloc-bench/out/bench +RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file From 98bbcda9610e3885e87ea36430c2f357738d4530 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 17:33:53 +0100 Subject: [PATCH 02/29] WIP --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e8ccb5c67..13c2636a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -38,7 +38,7 @@ jobs: # Uploads the benchmark results as an artifact - name: Upload Benchmark Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-results path: benchres.csv From 7227e2724fff485e0888f6c48db58d9f14c765c4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 17:35:59 +0100 Subject: [PATCH 03/29] WIP --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 13c2636a4..ab01fdc85 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,7 +3,7 @@ name: snmalloc Benchmarking CI # The following should ensure that the workflow only runs a single set of actions # for each PR. But it will not apply this to pushes to the main branch. concurrency: - group: ${{ github.ref }} + group: benchmarking${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} # Controls when the workflow will run From 5b786da5477f309127f2fa65de267266c7dc6649 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 09:02:13 +0100 Subject: [PATCH 04/29] Add caching --- .github/workflows/benchmark.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ab01fdc85..91bb0692b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,10 +26,21 @@ jobs: - name: Checkout uses: actions/checkout@v3 - # Builds the Docker image and runs the benchmarks - - name: Run Benchmarks - run: | - docker build -t snmalloc-bench -f benchmark/Dockerfile . --build-arg benchs=allt + # Setup docker buildx + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and Push Docker Image + uses: docker/build-push-action@v4 + with: + context: . + file: benchmark/Dockerfile + push: false + tags: snmalloc-bench + build-args: | + benchs=allt + cache-from: type=gha + cache-to: type=gha,mode=max # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results From e6965adea8a16875cfaba7066b24a614d4ae9d15 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:13:49 +0100 Subject: [PATCH 05/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 91bb0692b..aaa57f309 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,6 +31,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build and Push Docker Image + id: build uses: docker/build-push-action@v4 with: context: . @@ -45,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d snmalloc-bench`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 31718dbb9a0281a09ec0553145cee572d9edb67b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:17:49 +0100 Subject: [PATCH 06/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index aaa57f309..f37a3e5c0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 43b7f4e75ca117a3fa3ebf3545555c8d68d8ceb5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:18:00 +0100 Subject: [PATCH 07/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f37a3e5c0..4f7413fd4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,7 +39,7 @@ jobs: push: false tags: snmalloc-bench build-args: | - benchs=allt + benchs=cfrac cache-from: type=gha cache-to: type=gha,mode=max From 55fadf71534007231c7b0d2f9d13e4f4d57232da Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:21:40 +0100 Subject: [PATCH 08/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4f7413fd4..c889f423f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build and Push Docker Image - id: build + id: docker_build uses: docker/build-push-action@v4 with: context: . @@ -46,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 3a8acdd170983045c2e5bb428ffae733041b8273 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:23:34 +0100 Subject: [PATCH 09/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c889f423f..ef6f17f71 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 7a709a79ab98abe8ee9da8f1380a6220f72c5909 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:26:06 +0100 Subject: [PATCH 10/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ef6f17f71..cf2871d07 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -36,7 +36,7 @@ jobs: with: context: . file: benchmark/Dockerfile - push: false + push: true tags: snmalloc-bench build-args: | benchs=cfrac From 0b479c063246f69e145f52c0ed0eca2b31b65a53 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:32:43 +0100 Subject: [PATCH 11/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cf2871d07..ef6f17f71 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -36,7 +36,7 @@ jobs: with: context: . file: benchmark/Dockerfile - push: true + push: false tags: snmalloc-bench build-args: | benchs=cfrac From 5a20692ddce5f8cb783603db5d9bbbee5ad0c7c4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:42:09 +0100 Subject: [PATCH 12/29] Alter connecting to image. --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ef6f17f71..f2c6b0003 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -37,6 +37,7 @@ jobs: context: . file: benchmark/Dockerfile push: false + load: true tags: snmalloc-bench build-args: | benchs=cfrac From e314c10e7fc2c5a16c5cb41b9ad4e1c5931dd83d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:45:18 +0100 Subject: [PATCH 13/29] Investigate usage --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f2c6b0003..80451f702 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,7 +40,7 @@ jobs: load: true tags: snmalloc-bench build-args: | - benchs=cfrac + benchs=allt cache-from: type=gha cache-to: type=gha,mode=max From 9e6b1eb7af6ca14c9940c3dd899716858b86fa1b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:50:14 +0100 Subject: [PATCH 14/29] Investigate usage --- benchmark/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 0c72f068c..965c28d9c 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -34,4 +34,6 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file +RUN ../../bench.sh --external=/allocs.txt $benchs + +RUN du -m /mimalloc-bench \ No newline at end of file From c9333087bc7bc02e9349bb0246e249c3cff31232 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:50:54 +0100 Subject: [PATCH 15/29] Investigate usage --- benchmark/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 965c28d9c..230bb4564 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -19,6 +19,9 @@ RUN ./build-bench-env.sh rocksdb RUN ./build-bench-env.sh lean +# Tidy up apt cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt # Build allocator From b5efe7bd5e9558b15b546c31dbe1587d396f3d57 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:02:22 +0100 Subject: [PATCH 16/29] Investigate usage --- benchmark/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 230bb4564..8a81196c8 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -22,10 +22,12 @@ RUN ./build-bench-env.sh lean # Tidy up apt cache RUN apt-get clean && rm -rf /var/lib/apt/lists/* +# Remove rocksdb .o files to compact image +RUN find /mimalloc-bench/extern/rocksdb -name "*.o" -delete + RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt # Build allocator -ARG allocator=mi RUN mkdir -p /snmalloc COPY . /snmalloc From 244c07900c14484000f30b39988721f1d8cb8f86 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:06:34 +0100 Subject: [PATCH 17/29] Compacting. --- benchmark/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 8a81196c8..91285711d 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -23,6 +23,7 @@ RUN ./build-bench-env.sh lean RUN apt-get clean && rm -rf /var/lib/apt/lists/* # Remove rocksdb .o files to compact image +RUN apt-get install findutils RUN find /mimalloc-bench/extern/rocksdb -name "*.o" -delete RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt From bfb665d866f47628534f8db6bdb4b702251765b2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:08:33 +0100 Subject: [PATCH 18/29] Compacting. --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 91285711d..7e2b54d35 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* # Remove rocksdb .o files to compact image RUN apt-get install findutils -RUN find /mimalloc-bench/extern/rocksdb -name "*.o" -delete +RUN find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt From 5aff3a3aa26d3bcb883a87a178e938c27bfda2aa Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:13:51 +0100 Subject: [PATCH 19/29] Compacting. --- benchmark/Dockerfile | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 7e2b54d35..cf3ee4a51 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -10,21 +10,19 @@ WORKDIR /mimalloc-bench # Install dependencies RUN ./build-bench-env.sh packages +# Tidy up apt cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + # Build benchmarks RUN ./build-bench-env.sh bench RUN ./build-bench-env.sh redis -RUN ./build-bench-env.sh rocksdb - -RUN ./build-bench-env.sh lean +RUN ./build-bench-env.sh rocksdb \ + && find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete -# Tidy up apt cache -RUN apt-get clean && rm -rf /var/lib/apt/lists/* - -# Remove rocksdb .o files to compact image -RUN apt-get install findutils -RUN find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete +RUN ./build-bench-env.sh lean \ + && find /mimalloc-bench/extern/lean -name "*.o" -delete RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt @@ -40,6 +38,4 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs - -RUN du -m /mimalloc-bench \ No newline at end of file +RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file From 84dd7ae9e24cfb20f96d064004353eb889a22921 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 12:53:26 +0100 Subject: [PATCH 20/29] Apply suggestion from @achamayou Co-authored-by: Amaury Chamayou --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 80451f702..728753d52 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,7 +19,7 @@ on: jobs: benchmark: - runs-on: ubuntu-latest + runs-on: [self-hosted, 1ES.Pool=snmalloc-perf] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it From 6a280740fc9b81da6bfbcbe511d9bc485a3b649d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:02:38 +0100 Subject: [PATCH 21/29] Adjust output for bencher.dev --- .github/workflows/benchmark.yml | 4 ++-- benchmark/Dockerfile | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 728753d52..2fa62d8f6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,11 +47,11 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/results.json . # Uploads the benchmark results as an artifact - name: Upload Benchmark Results uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchres.csv + path: results.json diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index cf3ee4a51..67be3c9de 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -38,4 +38,6 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file +RUN ../../bench.sh --external=/allocs.txt $benchs -r=2 + +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From bee8d8dff410af2c64c38d82292fee03c86fb176 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:15:08 +0100 Subject: [PATCH 22/29] Add script --- benchmark/bencher.dev.py | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 benchmark/bencher.dev.py diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py new file mode 100644 index 000000000..ba6bb3046 --- /dev/null +++ b/benchmark/bencher.dev.py @@ -0,0 +1,75 @@ +# This script is adapted from the mimalloc-bench project. +# It converts the benchmark outputs to the format required by bencher.dev. + +import re +import sys +import collections +try: + import numpy as np +except ImportError: + print('You need to install numpy.') + sys.exit(1) + +if len(sys.argv) != 2: + print('Usage: %s benchres.csv' % sys.argv[0]) + print('Where benchres.csv is the output of the benchmark script. I.e.') + print(' mimalloc-bench/out/bench/benchres.csv') + print() + print('The script generates a single file for submission to bencher.dev.') + sys.exit(1) + +parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+)') +data = [] +test_names = set() + +# read in the data +with open(sys.argv[1]) as f: + for l in f.readlines(): + match = parse_line.search(l) + if not match: + continue + test_name, alloc_name, time_string, memory = match.groups() + time_split = time_string.split(':') + time_taken = 0 + test_names.add(test_name) + if len(time_split) == 2: + time_taken = int(time_split[0]) * 60 + float(time_split[1]) + else: + time_taken = float(time_split[0]) + data.append({"Benchmark":test_name, "Allocator":alloc_name, "Time":time_taken, "Memory":int(memory)}) + +# Output data in json of the form +# +# { +# "":{ +# "memory":{ +# "value": +# "high-value": +# "low-value": +# } +# "time":{ +# "value": +# "high-value": +# "low-value": +# } +# } +# } + +import json + +output = {} +for test_name in test_names: + output[test_name] = { + "memory": { + "value": float(np.mean([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Memory"] for d in data if d["Benchmark"] == test_name])), + }, + "time": { + "value": float(np.mean([d["Time"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Time"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Time"] for d in data if d["Benchmark"] == test_name])), + } + } + +print(json.dumps(output, indent=2)) \ No newline at end of file From 87671378b4f9fa0e73ce634931caf3c099dcd730 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:24:34 +0100 Subject: [PATCH 23/29] Adjust python deps --- .github/workflows/benchmark.yml | 15 ++++++++++++++- benchmark/Dockerfile | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2fa62d8f6..493897424 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,7 +40,8 @@ jobs: load: true tags: snmalloc-bench build-args: | - benchs=allt + benchs=cfrac + repeats=1 cache-from: type=gha cache-to: type=gha,mode=max @@ -55,3 +56,15 @@ jobs: with: name: benchmark-results path: results.json + + # Upload to graphing service + - uses: bencherdev/bencher@main + - name: Upload benchmark results to Bencher + run: | + bencher run \ + --project snmalloc \ + --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ + --branch main \ + --adapter json \ + --err \ + --file results.json \ No newline at end of file diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 67be3c9de..bb9674398 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -37,7 +37,10 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt +ARG repeats=1 WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs -r=2 +RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats +RUN apt-get install -y python3 python3-pip +RUN pip3 install --no-cache-dir numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From fef6300a3b7f13064a615027c4c70cb64b0465dd Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:31:53 +0100 Subject: [PATCH 24/29] Adjust python deps --- benchmark/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index bb9674398..3fb49c250 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,6 +41,5 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN apt-get install -y python3 python3-pip RUN pip3 install --no-cache-dir numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From c4a8d3edee67144926027654860a73ac8fb726f2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:39:01 +0100 Subject: [PATCH 25/29] Adjust python deps --- benchmark/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 3fb49c250..098b062be 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,5 +41,6 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats +RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-pip RUN pip3 install --no-cache-dir numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From 93ad1c792ac0137101aabbd053c1fabd1b5691d5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:45:10 +0100 Subject: [PATCH 26/29] Adjust python deps --- .github/workflows/benchmark.yml | 2 +- benchmark/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 493897424..249d4a689 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -64,7 +64,7 @@ jobs: bencher run \ --project snmalloc \ --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ - --branch main \ + --branch ${{ github.ref_name }} \ --adapter json \ --err \ --file results.json \ No newline at end of file diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 098b062be..9db5a55a2 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,6 +41,5 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-pip -RUN pip3 install --no-cache-dir numpy +RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From 1d132ac94b077a973c7784854dd1a3fa94bce9b8 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:54:10 +0100 Subject: [PATCH 27/29] Fix path --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 9db5a55a2..01ff868fa 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -42,4 +42,4 @@ WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy -RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > results.json \ No newline at end of file From e7a44e5817c9211e8f450eda124dd4d660f8908e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 15:37:20 +0100 Subject: [PATCH 28/29] Fix path --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 01ff868fa..979a4c1fc 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -42,4 +42,4 @@ WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy -RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > results.json \ No newline at end of file +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > /results.json \ No newline at end of file From 73dee9b902338a4f218f69d08d4edc898ddc0823 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 15:48:39 +0100 Subject: [PATCH 29/29] Move around deps. --- benchmark/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 979a4c1fc..2e70c974f 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:24.04 # Pull mimalloc-bench -RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates +RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates python3-numpy RUN git clone https://github.com/daanx/mimalloc-bench &&\ cd mimalloc-bench && \ git reset --hard ffa530dbbe046532dfcb4a1b58ffc06e144aee60 @@ -41,5 +41,4 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > /results.json \ No newline at end of file