From da5a310e34d6cbf6ff0b5dd758cf4ab56e6ebb94 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 17:32:30 +0100 Subject: [PATCH 01/40] Benchmarking automatically --- .github/workflows/benchmark.yml | 44 +++++++++++++++++++++++++++++++++ benchmark/Dockerfile | 37 +++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/Dockerfile diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..e8ccb5c67 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,44 @@ +name: snmalloc Benchmarking CI + +# The following should ensure that the workflow only runs a single set of actions +# for each PR. But it will not apply this to pushes to the main branch. +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ main ] + pull_request: + branches: [ main ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout + uses: actions/checkout@v3 + + # Builds the Docker image and runs the benchmarks + - name: Run Benchmarks + run: | + docker build -t snmalloc-bench -f benchmark/Dockerfile . --build-arg benchs=allt + + # Extracts the benchmark results from the Docker container + - name: Extract Benchmark Results + run: | + docker cp `docker run -d snmalloc-bench`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + + # Uploads the benchmark results as an artifact + - name: Upload Benchmark Results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: benchres.csv diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile new file mode 100644 index 000000000..0c72f068c --- /dev/null +++ b/benchmark/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 + +# Pull mimalloc-bench +RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates +RUN git clone https://github.com/daanx/mimalloc-bench &&\ + cd mimalloc-bench && \ + git reset --hard ffa530dbbe046532dfcb4a1b58ffc06e144aee60 + +WORKDIR /mimalloc-bench +# Install dependencies +RUN ./build-bench-env.sh packages + +# Build benchmarks +RUN ./build-bench-env.sh bench + +RUN ./build-bench-env.sh redis + +RUN ./build-bench-env.sh rocksdb + +RUN ./build-bench-env.sh lean + +RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt + +# Build allocator +ARG allocator=mi +RUN mkdir -p /snmalloc +COPY . /snmalloc + +RUN mkdir -p /snmalloc/build +WORKDIR /snmalloc/build +RUN cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. +RUN ninja libsnmallocshim.so + +# Run benchmarks +ARG benchs=allt +WORKDIR /mimalloc-bench/out/bench +RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file From 98bbcda9610e3885e87ea36430c2f357738d4530 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 17:33:53 +0100 Subject: [PATCH 02/40] WIP --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e8ccb5c67..13c2636a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -38,7 +38,7 @@ jobs: # Uploads the benchmark results as an artifact - name: Upload Benchmark Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-results path: benchres.csv From 7227e2724fff485e0888f6c48db58d9f14c765c4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 17:35:59 +0100 Subject: [PATCH 03/40] WIP --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 13c2636a4..ab01fdc85 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,7 +3,7 @@ name: snmalloc Benchmarking CI # The following should ensure that the workflow only runs a single set of actions # for each PR. But it will not apply this to pushes to the main branch. concurrency: - group: ${{ github.ref }} + group: benchmarking${{ github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} # Controls when the workflow will run From 5b786da5477f309127f2fa65de267266c7dc6649 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 09:02:13 +0100 Subject: [PATCH 04/40] Add caching --- .github/workflows/benchmark.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ab01fdc85..91bb0692b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,10 +26,21 @@ jobs: - name: Checkout uses: actions/checkout@v3 - # Builds the Docker image and runs the benchmarks - - name: Run Benchmarks - run: | - docker build -t snmalloc-bench -f benchmark/Dockerfile . --build-arg benchs=allt + # Setup docker buildx + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and Push Docker Image + uses: docker/build-push-action@v4 + with: + context: . + file: benchmark/Dockerfile + push: false + tags: snmalloc-bench + build-args: | + benchs=allt + cache-from: type=gha + cache-to: type=gha,mode=max # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results From e6965adea8a16875cfaba7066b24a614d4ae9d15 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:13:49 +0100 Subject: [PATCH 05/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 91bb0692b..aaa57f309 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,6 +31,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build and Push Docker Image + id: build uses: docker/build-push-action@v4 with: context: . @@ -45,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d snmalloc-bench`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 31718dbb9a0281a09ec0553145cee572d9edb67b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:17:49 +0100 Subject: [PATCH 06/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index aaa57f309..f37a3e5c0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 43b7f4e75ca117a3fa3ebf3545555c8d68d8ceb5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:18:00 +0100 Subject: [PATCH 07/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f37a3e5c0..4f7413fd4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,7 +39,7 @@ jobs: push: false tags: snmalloc-bench build-args: | - benchs=allt + benchs=cfrac cache-from: type=gha cache-to: type=gha,mode=max From 55fadf71534007231c7b0d2f9d13e4f4d57232da Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:21:40 +0100 Subject: [PATCH 08/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4f7413fd4..c889f423f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build and Push Docker Image - id: build + id: docker_build uses: docker/build-push-action@v4 with: context: . @@ -46,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${{steps.build.outputs.imageid}}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 3a8acdd170983045c2e5bb428ffae733041b8273 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:23:34 +0100 Subject: [PATCH 09/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c889f423f..ef6f17f71 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv # Uploads the benchmark results as an artifact - name: Upload Benchmark Results From 7a709a79ab98abe8ee9da8f1380a6220f72c5909 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:26:06 +0100 Subject: [PATCH 10/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ef6f17f71..cf2871d07 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -36,7 +36,7 @@ jobs: with: context: . file: benchmark/Dockerfile - push: false + push: true tags: snmalloc-bench build-args: | benchs=cfrac From 0b479c063246f69e145f52c0ed0eca2b31b65a53 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:32:43 +0100 Subject: [PATCH 11/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cf2871d07..ef6f17f71 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -36,7 +36,7 @@ jobs: with: context: . file: benchmark/Dockerfile - push: true + push: false tags: snmalloc-bench build-args: | benchs=cfrac From 5a20692ddce5f8cb783603db5d9bbbee5ad0c7c4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:42:09 +0100 Subject: [PATCH 12/40] Alter connecting to image. --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ef6f17f71..f2c6b0003 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -37,6 +37,7 @@ jobs: context: . file: benchmark/Dockerfile push: false + load: true tags: snmalloc-bench build-args: | benchs=cfrac From e314c10e7fc2c5a16c5cb41b9ad4e1c5931dd83d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:45:18 +0100 Subject: [PATCH 13/40] Investigate usage --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f2c6b0003..80451f702 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,7 +40,7 @@ jobs: load: true tags: snmalloc-bench build-args: | - benchs=cfrac + benchs=allt cache-from: type=gha cache-to: type=gha,mode=max From 9e6b1eb7af6ca14c9940c3dd899716858b86fa1b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:50:14 +0100 Subject: [PATCH 14/40] Investigate usage --- benchmark/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 0c72f068c..965c28d9c 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -34,4 +34,6 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file +RUN ../../bench.sh --external=/allocs.txt $benchs + +RUN du -m /mimalloc-bench \ No newline at end of file From c9333087bc7bc02e9349bb0246e249c3cff31232 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 10:50:54 +0100 Subject: [PATCH 15/40] Investigate usage --- benchmark/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 965c28d9c..230bb4564 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -19,6 +19,9 @@ RUN ./build-bench-env.sh rocksdb RUN ./build-bench-env.sh lean +# Tidy up apt cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt # Build allocator From b5efe7bd5e9558b15b546c31dbe1587d396f3d57 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:02:22 +0100 Subject: [PATCH 16/40] Investigate usage --- benchmark/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 230bb4564..8a81196c8 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -22,10 +22,12 @@ RUN ./build-bench-env.sh lean # Tidy up apt cache RUN apt-get clean && rm -rf /var/lib/apt/lists/* +# Remove rocksdb .o files to compact image +RUN find /mimalloc-bench/extern/rocksdb -name "*.o" -delete + RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt # Build allocator -ARG allocator=mi RUN mkdir -p /snmalloc COPY . /snmalloc From 244c07900c14484000f30b39988721f1d8cb8f86 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:06:34 +0100 Subject: [PATCH 17/40] Compacting. --- benchmark/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 8a81196c8..91285711d 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -23,6 +23,7 @@ RUN ./build-bench-env.sh lean RUN apt-get clean && rm -rf /var/lib/apt/lists/* # Remove rocksdb .o files to compact image +RUN apt-get install findutils RUN find /mimalloc-bench/extern/rocksdb -name "*.o" -delete RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt From bfb665d866f47628534f8db6bdb4b702251765b2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:08:33 +0100 Subject: [PATCH 18/40] Compacting. --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 91285711d..7e2b54d35 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* # Remove rocksdb .o files to compact image RUN apt-get install findutils -RUN find /mimalloc-bench/extern/rocksdb -name "*.o" -delete +RUN find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt From 5aff3a3aa26d3bcb883a87a178e938c27bfda2aa Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 11:13:51 +0100 Subject: [PATCH 19/40] Compacting. --- benchmark/Dockerfile | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 7e2b54d35..cf3ee4a51 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -10,21 +10,19 @@ WORKDIR /mimalloc-bench # Install dependencies RUN ./build-bench-env.sh packages +# Tidy up apt cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + # Build benchmarks RUN ./build-bench-env.sh bench RUN ./build-bench-env.sh redis -RUN ./build-bench-env.sh rocksdb - -RUN ./build-bench-env.sh lean +RUN ./build-bench-env.sh rocksdb \ + && find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete -# Tidy up apt cache -RUN apt-get clean && rm -rf /var/lib/apt/lists/* - -# Remove rocksdb .o files to compact image -RUN apt-get install findutils -RUN find /mimalloc-bench/extern/rocksdb-8.1.1 -name "*.o" -delete +RUN ./build-bench-env.sh lean \ + && find /mimalloc-bench/extern/lean -name "*.o" -delete RUN echo "sn /snmalloc/build/libsnmallocshim.so" > /allocs.txt @@ -40,6 +38,4 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs - -RUN du -m /mimalloc-bench \ No newline at end of file +RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file From 84dd7ae9e24cfb20f96d064004353eb889a22921 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 12:53:26 +0100 Subject: [PATCH 20/40] Apply suggestion from @achamayou Co-authored-by: Amaury Chamayou --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 80451f702..728753d52 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,7 +19,7 @@ on: jobs: benchmark: - runs-on: ubuntu-latest + runs-on: [self-hosted, 1ES.Pool=snmalloc-perf] steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it From 6a280740fc9b81da6bfbcbe511d9bc485a3b649d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:02:38 +0100 Subject: [PATCH 21/40] Adjust output for bencher.dev --- .github/workflows/benchmark.yml | 4 ++-- benchmark/Dockerfile | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 728753d52..2fa62d8f6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,11 +47,11 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/mimalloc-bench/out/bench/benchres.csv benchres.csv + docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/results.json . # Uploads the benchmark results as an artifact - name: Upload Benchmark Results uses: actions/upload-artifact@v4 with: name: benchmark-results - path: benchres.csv + path: results.json diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index cf3ee4a51..67be3c9de 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -38,4 +38,6 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs \ No newline at end of file +RUN ../../bench.sh --external=/allocs.txt $benchs -r=2 + +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From bee8d8dff410af2c64c38d82292fee03c86fb176 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:15:08 +0100 Subject: [PATCH 22/40] Add script --- benchmark/bencher.dev.py | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 benchmark/bencher.dev.py diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py new file mode 100644 index 000000000..ba6bb3046 --- /dev/null +++ b/benchmark/bencher.dev.py @@ -0,0 +1,75 @@ +# This script is adapted from the mimalloc-bench project. +# It converts the benchmark outputs to the format required by bencher.dev. + +import re +import sys +import collections +try: + import numpy as np +except ImportError: + print('You need to install numpy.') + sys.exit(1) + +if len(sys.argv) != 2: + print('Usage: %s benchres.csv' % sys.argv[0]) + print('Where benchres.csv is the output of the benchmark script. I.e.') + print(' mimalloc-bench/out/bench/benchres.csv') + print() + print('The script generates a single file for submission to bencher.dev.') + sys.exit(1) + +parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+)') +data = [] +test_names = set() + +# read in the data +with open(sys.argv[1]) as f: + for l in f.readlines(): + match = parse_line.search(l) + if not match: + continue + test_name, alloc_name, time_string, memory = match.groups() + time_split = time_string.split(':') + time_taken = 0 + test_names.add(test_name) + if len(time_split) == 2: + time_taken = int(time_split[0]) * 60 + float(time_split[1]) + else: + time_taken = float(time_split[0]) + data.append({"Benchmark":test_name, "Allocator":alloc_name, "Time":time_taken, "Memory":int(memory)}) + +# Output data in json of the form +# +# { +# "":{ +# "memory":{ +# "value": +# "high-value": +# "low-value": +# } +# "time":{ +# "value": +# "high-value": +# "low-value": +# } +# } +# } + +import json + +output = {} +for test_name in test_names: + output[test_name] = { + "memory": { + "value": float(np.mean([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Memory"] for d in data if d["Benchmark"] == test_name])), + }, + "time": { + "value": float(np.mean([d["Time"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Time"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Time"] for d in data if d["Benchmark"] == test_name])), + } + } + +print(json.dumps(output, indent=2)) \ No newline at end of file From 87671378b4f9fa0e73ce634931caf3c099dcd730 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:24:34 +0100 Subject: [PATCH 23/40] Adjust python deps --- .github/workflows/benchmark.yml | 15 ++++++++++++++- benchmark/Dockerfile | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2fa62d8f6..493897424 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,7 +40,8 @@ jobs: load: true tags: snmalloc-bench build-args: | - benchs=allt + benchs=cfrac + repeats=1 cache-from: type=gha cache-to: type=gha,mode=max @@ -55,3 +56,15 @@ jobs: with: name: benchmark-results path: results.json + + # Upload to graphing service + - uses: bencherdev/bencher@main + - name: Upload benchmark results to Bencher + run: | + bencher run \ + --project snmalloc \ + --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ + --branch main \ + --adapter json \ + --err \ + --file results.json \ No newline at end of file diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 67be3c9de..bb9674398 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -37,7 +37,10 @@ RUN ninja libsnmallocshim.so # Run benchmarks ARG benchs=allt +ARG repeats=1 WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh --external=/allocs.txt $benchs -r=2 +RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats +RUN apt-get install -y python3 python3-pip +RUN pip3 install --no-cache-dir numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From fef6300a3b7f13064a615027c4c70cb64b0465dd Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:31:53 +0100 Subject: [PATCH 24/40] Adjust python deps --- benchmark/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index bb9674398..3fb49c250 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,6 +41,5 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN apt-get install -y python3 python3-pip RUN pip3 install --no-cache-dir numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From c4a8d3edee67144926027654860a73ac8fb726f2 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:39:01 +0100 Subject: [PATCH 25/40] Adjust python deps --- benchmark/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 3fb49c250..098b062be 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,5 +41,6 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats +RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-pip RUN pip3 install --no-cache-dir numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From 93ad1c792ac0137101aabbd053c1fabd1b5691d5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:45:10 +0100 Subject: [PATCH 26/40] Adjust python deps --- .github/workflows/benchmark.yml | 2 +- benchmark/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 493897424..249d4a689 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -64,7 +64,7 @@ jobs: bencher run \ --project snmalloc \ --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ - --branch main \ + --branch ${{ github.ref_name }} \ --adapter json \ --err \ --file results.json \ No newline at end of file diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 098b062be..9db5a55a2 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,6 +41,5 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-pip -RUN pip3 install --no-cache-dir numpy +RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file From 1d132ac94b077a973c7784854dd1a3fa94bce9b8 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 14:54:10 +0100 Subject: [PATCH 27/40] Fix path --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 9db5a55a2..01ff868fa 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -42,4 +42,4 @@ WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy -RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/benchres.csv > results.json \ No newline at end of file +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > results.json \ No newline at end of file From e7a44e5817c9211e8f450eda124dd4d660f8908e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 15:37:20 +0100 Subject: [PATCH 28/40] Fix path --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 01ff868fa..979a4c1fc 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -42,4 +42,4 @@ WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy -RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > results.json \ No newline at end of file +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > /results.json \ No newline at end of file From 73dee9b902338a4f218f69d08d4edc898ddc0823 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 15:48:39 +0100 Subject: [PATCH 29/40] Move around deps. --- benchmark/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 979a4c1fc..2e70c974f 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:24.04 # Pull mimalloc-bench -RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates +RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates python3-numpy RUN git clone https://github.com/daanx/mimalloc-bench &&\ cd mimalloc-bench && \ git reset --hard ffa530dbbe046532dfcb4a1b58ffc06e144aee60 @@ -41,5 +41,4 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN apt-get update && apt-get install --no-install-recommends -y python3 python3-numpy RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > /results.json \ No newline at end of file From 47f46ff44d228299b8e531bdf732f85ad087a486 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 21:10:29 +0100 Subject: [PATCH 30/40] Set to correct benchmarking. --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 249d4a689..142c83c57 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -40,8 +40,8 @@ jobs: load: true tags: snmalloc-bench build-args: | - benchs=cfrac - repeats=1 + benchs=allt + repeats=5 cache-from: type=gha cache-to: type=gha,mode=max From 20a234625266bd5be6c53b710335f13e245eb99c Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 22:11:18 +0100 Subject: [PATCH 31/40] Change name --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 142c83c57..b6fefbcd1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -30,7 +30,7 @@ jobs: - name: Setup Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build and Push Docker Image + - name: Build and run benchmarks in Docker id: docker_build uses: docker/build-push-action@v4 with: From 37d76246083b8c7cb8fa110747b26269dd45885b Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Wed, 6 Aug 2025 22:15:03 +0100 Subject: [PATCH 32/40] Add cron --- .github/workflows/benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b6fefbcd1..42dc59731 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,6 +8,8 @@ concurrency: # Controls when the workflow will run on: + schedule: + - cron: "0 0 * * 1" # Runs every Monday at midnight UTC # Triggers the workflow on push or pull request events but only for the master branch push: branches: [ main ] From c9641727b1d1c6ebd3ca1991076f52263bee2fe4 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 7 Aug 2025 09:19:39 +0100 Subject: [PATCH 33/40] Update script to handle multiple allocators --- .github/workflows/benchmark.yml | 6 +++--- benchmark/Dockerfile | 2 +- benchmark/bencher.dev.py | 32 ++++++++++++++++++-------------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 42dc59731..c4b3bfb15 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -50,14 +50,14 @@ jobs: # Extracts the benchmark results from the Docker container - name: Extract Benchmark Results run: | - docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/results.json . + docker cp `docker run -d ${{ steps.docker_build.outputs.imageid }}`:/bencher.dev.sn.json . # Uploads the benchmark results as an artifact - name: Upload Benchmark Results uses: actions/upload-artifact@v4 with: name: benchmark-results - path: results.json + path: bencher.dev.sn.json # Upload to graphing service - uses: bencherdev/bencher@main @@ -69,4 +69,4 @@ jobs: --branch ${{ github.ref_name }} \ --adapter json \ --err \ - --file results.json \ No newline at end of file + --file bencher.dev.sn.json \ No newline at end of file diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 2e70c974f..4c2a877eb 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,4 +41,4 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats -RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv > /results.json \ No newline at end of file +RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv \ No newline at end of file diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py index ba6bb3046..a442ad828 100644 --- a/benchmark/bencher.dev.py +++ b/benchmark/bencher.dev.py @@ -57,19 +57,23 @@ import json -output = {} -for test_name in test_names: - output[test_name] = { - "memory": { - "value": float(np.mean([d["Memory"] for d in data if d["Benchmark"] == test_name])), - "high-value": float(np.max([d["Memory"] for d in data if d["Benchmark"] == test_name])), - "low-value": float(np.min([d["Memory"] for d in data if d["Benchmark"] == test_name])), - }, - "time": { - "value": float(np.mean([d["Time"] for d in data if d["Benchmark"] == test_name])), - "high-value": float(np.max([d["Time"] for d in data if d["Benchmark"] == test_name])), - "low-value": float(np.min([d["Time"] for d in data if d["Benchmark"] == test_name])), +for alloc in set(d["Allocator"] for d in data if d["Benchmark"] == test_name): + output = {} + for test_name in test_names: + output[test_name] = { + "memory": { + "value": float(np.mean([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Memory"] for d in data if d["Benchmark"] == test_name])), + }, + "time": { + "value": float(np.mean([d["Time"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Time"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Time"] for d in data if d["Benchmark"] == test_name])), + } } - } + result = json.dumps(output, indent=2) + with open(f"bencher.dev.{alloc}.json", "w") as f: + f.write(result) + print(f"Output written to bencher.dev.{alloc}.json") -print(json.dumps(output, indent=2)) \ No newline at end of file From d2e049a59e4d557a9ff109777aa738d55fa67dc3 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Thu, 7 Aug 2025 09:39:04 +0100 Subject: [PATCH 34/40] Put output in right place. --- benchmark/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 4c2a877eb..8a8c8cec2 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -41,4 +41,5 @@ ARG repeats=1 WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats +WORKDIR / RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv \ No newline at end of file From 83df8517cfb4086c5b5a9b260386f0af07fd0a9e Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 14:28:15 +0100 Subject: [PATCH 35/40] Refinements --- .github/workflows/benchmark.yml | 2 +- benchmark/bencher.dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c4b3bfb15..9466c3e4e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -66,7 +66,7 @@ jobs: bencher run \ --project snmalloc \ --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ - --branch ${{ github.ref_name }} \ + --branch main \ --adapter json \ --err \ --file bencher.dev.sn.json \ No newline at end of file diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py index a442ad828..a1fb55ea1 100644 --- a/benchmark/bencher.dev.py +++ b/benchmark/bencher.dev.py @@ -18,7 +18,7 @@ print('The script generates a single file for submission to bencher.dev.') sys.exit(1) -parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+)') +parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+)$') data = [] test_names = set() From dfc8a2c51d5b65a6d7c03c37f26f7bd08a39aab5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 17:15:57 +0100 Subject: [PATCH 36/40] refinement --- benchmark/bencher.dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py index a1fb55ea1..88fb32f73 100644 --- a/benchmark/bencher.dev.py +++ b/benchmark/bencher.dev.py @@ -18,7 +18,7 @@ print('The script generates a single file for submission to bencher.dev.') sys.exit(1) -parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+)$') +parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+) *$') data = [] test_names = set() From f501e3fd37884af6c144b6f3d02211da480d19a8 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 17:31:39 +0100 Subject: [PATCH 37/40] refinement --- benchmark/bencher.dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py index 88fb32f73..13529696a 100644 --- a/benchmark/bencher.dev.py +++ b/benchmark/bencher.dev.py @@ -18,7 +18,7 @@ print('The script generates a single file for submission to bencher.dev.') sys.exit(1) -parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+) *$') +parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+) [0-9:.]+ [0-9:.]+ [0-9:.]+ [0-9:.]+$') data = [] test_names = set() From b23d006532a479414f71200f3a7acb264c18ee4d Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 21:19:12 +0100 Subject: [PATCH 38/40] Script is now upstreamed to benchmarks. --- benchmark/Dockerfile | 4 +- benchmark/bencher.dev.py | 79 ---------------------------------------- 2 files changed, 2 insertions(+), 81 deletions(-) delete mode 100644 benchmark/bencher.dev.py diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index 8a8c8cec2..f2cbe50d4 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:24.04 RUN apt-get update && apt-get install -y --no-install-recommends git gpg ca-certificates python3-numpy RUN git clone https://github.com/daanx/mimalloc-bench &&\ cd mimalloc-bench && \ - git reset --hard ffa530dbbe046532dfcb4a1b58ffc06e144aee60 + git reset --hard a4ce904286365c7adfba54f0eea3a2df3fc95bd1 WORKDIR /mimalloc-bench # Install dependencies @@ -42,4 +42,4 @@ WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats WORKDIR / -RUN python3 /snmalloc/benchmark/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv \ No newline at end of file +RUN python3 /mimalloc-bench/script/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv \ No newline at end of file diff --git a/benchmark/bencher.dev.py b/benchmark/bencher.dev.py deleted file mode 100644 index 13529696a..000000000 --- a/benchmark/bencher.dev.py +++ /dev/null @@ -1,79 +0,0 @@ -# This script is adapted from the mimalloc-bench project. -# It converts the benchmark outputs to the format required by bencher.dev. - -import re -import sys -import collections -try: - import numpy as np -except ImportError: - print('You need to install numpy.') - sys.exit(1) - -if len(sys.argv) != 2: - print('Usage: %s benchres.csv' % sys.argv[0]) - print('Where benchres.csv is the output of the benchmark script. I.e.') - print(' mimalloc-bench/out/bench/benchres.csv') - print() - print('The script generates a single file for submission to bencher.dev.') - sys.exit(1) - -parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+) [0-9:.]+ [0-9:.]+ [0-9:.]+ [0-9:.]+$') -data = [] -test_names = set() - -# read in the data -with open(sys.argv[1]) as f: - for l in f.readlines(): - match = parse_line.search(l) - if not match: - continue - test_name, alloc_name, time_string, memory = match.groups() - time_split = time_string.split(':') - time_taken = 0 - test_names.add(test_name) - if len(time_split) == 2: - time_taken = int(time_split[0]) * 60 + float(time_split[1]) - else: - time_taken = float(time_split[0]) - data.append({"Benchmark":test_name, "Allocator":alloc_name, "Time":time_taken, "Memory":int(memory)}) - -# Output data in json of the form -# -# { -# "":{ -# "memory":{ -# "value": -# "high-value": -# "low-value": -# } -# "time":{ -# "value": -# "high-value": -# "low-value": -# } -# } -# } - -import json - -for alloc in set(d["Allocator"] for d in data if d["Benchmark"] == test_name): - output = {} - for test_name in test_names: - output[test_name] = { - "memory": { - "value": float(np.mean([d["Memory"] for d in data if d["Benchmark"] == test_name])), - "high-value": float(np.max([d["Memory"] for d in data if d["Benchmark"] == test_name])), - "low-value": float(np.min([d["Memory"] for d in data if d["Benchmark"] == test_name])), - }, - "time": { - "value": float(np.mean([d["Time"] for d in data if d["Benchmark"] == test_name])), - "high-value": float(np.max([d["Time"] for d in data if d["Benchmark"] == test_name])), - "low-value": float(np.min([d["Time"] for d in data if d["Benchmark"] == test_name])), - } - } - result = json.dumps(output, indent=2) - with open(f"bencher.dev.{alloc}.json", "w") as f: - f.write(result) - print(f"Output written to bencher.dev.{alloc}.json") - From 26f44e96b33fef72f6af1b4a23453dd4f57c3665 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 21:31:31 +0100 Subject: [PATCH 39/40] Disable GWP Asan jemalloc test --- src/test/func/jemalloc/jemalloc.cc | 113 +++++++++++++++-------------- 1 file changed, 60 insertions(+), 53 deletions(-) diff --git a/src/test/func/jemalloc/jemalloc.cc b/src/test/func/jemalloc/jemalloc.cc index e9d046112..a50bf38bd 100644 --- a/src/test/func/jemalloc/jemalloc.cc +++ b/src/test/func/jemalloc/jemalloc.cc @@ -1,65 +1,71 @@ -#include -#include -#include -#include -#include +#if defined(SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION) +int main() +{ + return 0; +} +#else +# include +# include +# include +# include +# include -#define SNMALLOC_NAME_MANGLE(a) our_##a -#undef SNMALLOC_NO_REALLOCARRAY -#undef SNMALLOC_NO_REALLOCARR -#define SNMALLOC_BOOTSTRAP_ALLOCATOR -#define SNMALLOC_JEMALLOC3_EXPERIMENTAL -#define SNMALLOC_JEMALLOC_NONSTANDARD -#include -#include +# define SNMALLOC_NAME_MANGLE(a) our_##a +# undef SNMALLOC_NO_REALLOCARRAY +# undef SNMALLOC_NO_REALLOCARR +# define SNMALLOC_BOOTSTRAP_ALLOCATOR +# define SNMALLOC_JEMALLOC3_EXPERIMENTAL +# define SNMALLOC_JEMALLOC_NONSTANDARD +# include +# include -#if __has_include() -# include -#endif +# if __has_include() +# include +# endif -#ifdef __FreeBSD__ +# ifdef __FreeBSD__ /** * Enable testing against the versions that we get from libc or elsewhere. * Enabled by default on FreeBSD where all of the jemalloc functions are * exported from libc. */ -# define TEST_JEMALLOC_MALLOCX -#endif +# define TEST_JEMALLOC_MALLOCX +# endif -#define OUR_MALLOCX_LG_ALIGN(la) (static_cast(la)) -#define OUR_MALLOCX_ZERO (one_at_bit(6)) +# define OUR_MALLOCX_LG_ALIGN(la) (static_cast(la)) +# define OUR_MALLOCX_ZERO (one_at_bit(6)) -#define OUR_ALLOCM_NO_MOVE (one_at_bit(7)) +# define OUR_ALLOCM_NO_MOVE (one_at_bit(7)) -#define OUR_ALLOCM_SUCCESS 0 -#define OUR_ALLOCM_ERR_OOM 1 -#define OUR_ALLOCM_ERR_NOT_MOVED 2 +# define OUR_ALLOCM_SUCCESS 0 +# define OUR_ALLOCM_ERR_OOM 1 +# define OUR_ALLOCM_ERR_NOT_MOVED 2 -#ifndef MALLOCX_LG_ALIGN -# define MALLOCX_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la) -#endif -#ifndef MALLOCX_ZERO -# define MALLOCX_ZERO OUR_MALLOCX_ZERO -#endif +# ifndef MALLOCX_LG_ALIGN +# define MALLOCX_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la) +# endif +# ifndef MALLOCX_ZERO +# define MALLOCX_ZERO OUR_MALLOCX_ZERO +# endif -#ifndef ALLOCM_LG_ALIGN -# define ALLOCM_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la) -#endif -#ifndef ALLOCM_ZERO -# define ALLOCM_ZERO OUR_MALLOCX_ZERO -#endif -#ifndef ALLOCM_NO_MOVE -# define ALLOCM_NO_MOVE OUR_ALLOCM_NO_MOVE -#endif -#ifndef ALLOCM_SUCCESS -# define ALLOCM_SUCCESS OUR_ALLOCM_SUCCESS -#endif -#ifndef ALLOCM_ERR_OOM -# define ALLOCM_ERR_OOM OUR_ALLOCM_ERR_OOM -#endif -#ifndef ALLOCM_ERR_NOT_MOVED -# define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED -#endif +# ifndef ALLOCM_LG_ALIGN +# define ALLOCM_LG_ALIGN(la) OUR_MALLOCX_LG_ALIGN(la) +# endif +# ifndef ALLOCM_ZERO +# define ALLOCM_ZERO OUR_MALLOCX_ZERO +# endif +# ifndef ALLOCM_NO_MOVE +# define ALLOCM_NO_MOVE OUR_ALLOCM_NO_MOVE +# endif +# ifndef ALLOCM_SUCCESS +# define ALLOCM_SUCCESS OUR_ALLOCM_SUCCESS +# endif +# ifndef ALLOCM_ERR_OOM +# define ALLOCM_ERR_OOM OUR_ALLOCM_ERR_OOM +# endif +# ifndef ALLOCM_ERR_NOT_MOVED +# define ALLOCM_ERR_NOT_MOVED OUR_ALLOCM_ERR_NOT_MOVED +# endif using namespace snmalloc; using namespace snmalloc::bits; @@ -335,21 +341,22 @@ int main() our_dallocm, our_nallocm>(); -#ifndef __PIC__ +# ifndef __PIC__ void* bootstrap = __je_bootstrap_malloc(42); if (bootstrap == nullptr) { printf("Failed to allocate from bootstrap malloc\n"); } __je_bootstrap_free(bootstrap); -#endif +# endif // These tests are for jemalloc compatibility and so should work with // jemalloc's implementation of these functions. If TEST_JEMALLOC is // defined then we try -#ifdef TEST_JEMALLOC_MALLOCX +# ifdef TEST_JEMALLOC_MALLOCX test_size(); test_zeroing(); test_xallocx(); -#endif +# endif } +#endif // SNMALLOC_ENABLE_GWP_ASAN_INTEGRATION \ No newline at end of file From 461c23f04f1c34e4711f67e7b8df2ce39579fdcb Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Sat, 9 Aug 2025 07:37:51 +0100 Subject: [PATCH 40/40] Update Dockerfile --- benchmark/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile index f2cbe50d4..76cd5aa17 100644 --- a/benchmark/Dockerfile +++ b/benchmark/Dockerfile @@ -42,4 +42,4 @@ WORKDIR /mimalloc-bench/out/bench RUN ../../bench.sh --external=/allocs.txt $benchs -r=$repeats WORKDIR / -RUN python3 /mimalloc-bench/script/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv \ No newline at end of file +RUN python3 /mimalloc-bench/scripts/bencher.dev.py /mimalloc-bench/out/bench/benchres.csv \ No newline at end of file