feat(server): Add on-demand heap profiling endpoints

.github/workflows/build.yml

@@ -46,7 +46,7 @@ jobs:
 
       - name: Build Binary
         run: |
-          cargo build --release --locked --target=${{ matrix.target }} --bin objectstore
+          cargo build --release --locked --target=${{ matrix.target }} --bin objectstore --features profiling
           cp target/${{ matrix.target }}/release/objectstore ./objectstore
 
       - name: Set up Docker Buildx

Cargo.toml

@@ -77,6 +77,7 @@ serde_yaml = "0.9.34-deprecated"
 sketches-ddsketch = "0.3.1"
 tempfile = "3.27.0"
 thiserror = "2.0.18"
+jemalloc_pprof = { version = "0.9.0", features = ["symbolize"] }
 tikv-jemallocator = { version = "0.7.0", features = ["background_threads", "override_allocator_on_supported_platforms"] }
 tikv-jemalloc-ctl = { version = "0.7.0", features = ["stats"] }
 tokio = "1.52.3"

README.md

@@ -243,6 +243,36 @@ cargo run -- run
 You can copy and save additional config files next to the examples in
 `objectstore-server/config`. All other files are ignored by git.
 
+### Heap Profiling
+
+Production release builds include on-demand heap profiling via jemalloc. It can
+be enabled and disabled through HTTP endpoints that are only reachable from
+loopback.
+
+To capture a heap profile from a running server, for example on localhost with
+default port:
+
+```sh
+# Enable sampling, let the workload run, then dump a profile
+curl -X POST http://localhost:8888/debug/pprof/enable
+curl http://localhost:8888/debug/pprof/heap > heap.pb.gz
+
+# Disable sampling when done
+curl -X POST http://localhost:8888/debug/pprof/disable
+```
+
+Analyze the profile dump with `go tool pprof`:
+
+```sh
+go tool pprof heap.pb.gz
+
+# To isolate growth between two snapshots, use the -base flag:
+go tool pprof -base before.pb.gz after.pb.gz
+```
+
+The sampling overhead is expected to be low, so profiling can be left enabled
+for an extended capture window safely.
+
 ### Tests
 
 To run tests:

objectstore-server/Cargo.toml

@@ -45,9 +45,13 @@ tower = { workspace = true }
 tower-http = { workspace = true, features = ["catch-panic", "metrics", "set-header", "trace"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
+jemalloc_pprof = { workspace = true, optional = true }
 tikv-jemallocator = { workspace = true }
 tikv-jemalloc-ctl = { workspace = true }
 
+[features]
+profiling = ["dep:jemalloc_pprof", "tikv-jemallocator/profiling"]
+
 [dev-dependencies]
 nix = { workspace = true, features = ["signal"] }
 objectstore-test = { workspace = true }

objectstore-server/src/endpoints/mod.rs

@@ -12,10 +12,12 @@ pub mod health;
 mod keda;
 mod multipart;
 mod objects;
+#[cfg(all(target_os = "linux", feature = "profiling"))]
+mod profiling;
 
 /// Returns `true` for internal endpoints that are exempt from metrics and concurrency limits.
 pub fn is_internal_route(route: &str) -> bool {
-    matches!(route, "/health" | "/ready" | "/keda")
+    matches!(route, "/health" | "/ready" | "/keda") || route.starts_with("/debug/")
 }
 
 /// Returns a router with all objectstore HTTP endpoints mounted.
@@ -28,8 +30,15 @@ pub fn routes() -> Router<ServiceState> {
         .merge(batch::router())
         .merge(multipart::router());
 
-    Router::new()
+    let router = Router::new()
         .merge(health::router())
         .merge(keda::router())
-        .nest("/v1/", routes_v1)
+        .nest("/v1/", routes_v1);
+
+    std::cfg_select! {
+        all(target_os = "linux", feature = "profiling") => {
+            router.merge(profiling::router())
+        }
+        _ => { router }
+    }
 }