Add required ci check for keynote-2 benchmark

.github/workflows/ci.yml

@@ -265,6 +265,56 @@ jobs:
           source ~/emsdk/emsdk_env.sh
           cargo ci test
 
+  keynote_bench:
+    needs: [lints]
+    name: Keynote Bench
+    runs-on: spacetimedb-new-runner-2
+    timeout-minutes: 60
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+    steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Set default rust toolchain
+        run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
+
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          save-if: false
+          prefix-key: v1
+
+      # Node 24 is the current Active LTS line.
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - uses: ./.github/actions/setup-pnpm
+        with:
+          run_install: true
+
+      - name: Run keynote-2 benchmark regression check
+        run: cargo ci keynote-bench
+
   lints:
     name: Lints
     runs-on: spacetimedb-new-runner-2

templates/keynote-2/src/cli.ts

@@ -5,7 +5,7 @@ import { CONNECTORS } from './connectors';
 import { runOne } from './core/runner';
 import type { TestCaseModule } from './tests/types';
 import { fileURLToPath } from 'node:url';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { RunResult } from './core/types.ts';
 import { parseBenchOptions } from './opts.ts';
 
@@ -153,12 +153,21 @@ function runPrep(): Promise<void> {
 
 const testDirUrl = new URL(`./tests/${testName}/`, import.meta.url);
 const testDirPath = fileURLToPath(testDirUrl);
-const runsDir = fileURLToPath(new URL('../runs/', import.meta.url));
+const runsDir = process.env.BENCH_RUNS_DIR
+  ? resolve(process.env.BENCH_RUNS_DIR)
+  : fileURLToPath(new URL('../runs/', import.meta.url));
 
-async function writeRunJson(payload: object, connectorName: string, alpha: number) {
+async function writeRunJson(
+  payload: object,
+  connectorName: string,
+  alpha: number,
+) {
   await mkdir(runsDir, { recursive: true });
   const ts = new Date().toISOString().replace(/[:.]/g, '-');
-  const outFile = join(runsDir, `${testName}-${connectorName}-a${alpha}-${ts}.json`);
+  const outFile = join(
+    runsDir,
+    `${testName}-${connectorName}-a${alpha}-${ts}.json`,
+  );
   await writeFile(outFile, JSON.stringify(payload, null, 2));
   console.log(`Wrote results to ${outFile}`);
   return outFile;

tools/ci/Cargo.toml

@@ -13,3 +13,4 @@ serde_json.workspace = true
 duct.workspace = true
 tempfile.workspace = true
 env_logger.workspace = true
+spacetimedb-guard.workspace = true

tools/ci/README.md

@@ -143,6 +143,21 @@ Usage: help [COMMAND]...
 
 - `subcommand`: Print help for the subcommand(s)
 
+### `keynote-bench`
+
+Runs the keynote benchmark as a CI performance regression gate.
+
+Builds release SpacetimeDB binaries, runs the keynote SpacetimeDB benchmark for 60 seconds, and fails if throughput is below 250K TPS.
+
+**Usage:**
+```bash
+Usage: keynote-bench
+```
+
+**Options:**
+
+- `--help`: Print help (see a summary with '-h')
+
 ### `update-flow`
 
 Tests the update flow

tools/ci/src/keynote_bench.rs

@@ -0,0 +1,216 @@
+#![allow(clippy::disallowed_macros)]
+
+use anyhow::{bail, ensure, Context, Result};
+use serde_json::Value;
+use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard};
+use std::{
+    env, fs,
+    path::{Path, PathBuf},
+    process::Command,
+};
+
+const DATABASE_NAME: &str = "test-1";
+const KEYNOTE_DIR: &str = "templates/keynote-2";
+const KEYNOTE_MODULE_DIR: &str = "templates/keynote-2/spacetimedb";
+const KEYNOTE_BINDINGS_DIR: &str = "templates/keynote-2/module_bindings";
+const MIN_TPS: f64 = 250_000.0;
+const BENCH_SECONDS: &str = "60";
+const BENCH_CONCURRENCY: &str = "64";
+const MAX_INFLIGHT_PER_WORKER: &str = "64";
+const SEED_ACCOUNTS: &str = "100000";
+const SEED_INITIAL_BALANCE: &str = "1000000000000";
+
+pub fn run() -> Result<()> {
+    build_typescript_sdk()?;
+    build_release_binaries()?;
+
+    let cli_path = ensure_binaries_built();
+    let server = SpacetimeDbGuard::spawn_in_temp_data_dir();
+    let cli_config_dir = tempfile::tempdir().context("failed to create temporary CLI config directory")?;
+    let cli_config_path = cli_config_dir.path().join("config.toml");
+
+    publish_module(&cli_path, &cli_config_path, &server.host_url)?;
+    generate_module_bindings(&cli_path, &cli_config_path)?;
+    seed_accounts(&cli_path, &cli_config_path, &server.host_url)?;
+
+    let runs_dir = tempfile::tempdir().context("failed to create temporary benchmark runs directory")?;
+    run_benchmark(&server.host_url, runs_dir.path())?;
+
+    let result_path = find_result_json(runs_dir.path())?;
+    let result_json = fs::read_to_string(&result_path)
+        .with_context(|| format!("failed to read benchmark result {}", result_path.display()))?;
+    let tps = result_tps(&result_json)?;
+
+    if tps < MIN_TPS {
+        eprintln!(
+            "Keynote perf regression: throughput {tps:.0} TPS < {MIN_TPS:.0} TPS\n\nResult JSON:\n{}",
+            result_json
+        );
+        bail!("keynote benchmark throughput is below threshold");
+    }
+
+    println!(
+        "Keynote perf check passed: throughput {tps:.0} TPS >= {MIN_TPS:.0} TPS ({})",
+        result_path.display()
+    );
+    Ok(())
+}
+
+fn build_typescript_sdk() -> Result<()> {
+    let mut cmd = Command::new("pnpm");
+    cmd.arg("build").current_dir("crates/bindings-typescript");
+    run_command(&mut cmd, "pnpm build in crates/bindings-typescript")
+}
+
+fn build_release_binaries() -> Result<()> {
+    eprintln!("Building spacetimedb-cli and spacetimedb-standalone (release)...");
+    let mut cmd = Command::new("cargo");
+    cmd.args([
+        "build",
+        "--release",
+        "-p",
+        "spacetimedb-cli",
+        "-p",
+        "spacetimedb-standalone",
+        "--features",
+        "spacetimedb-standalone/allow_loopback_http_for_tests",
+    ]);
+    remove_cargo_env_vars(&mut cmd);
+    run_command(&mut cmd, "cargo build --release spacetimedb-cli spacetimedb-standalone")
+}
+
+fn remove_cargo_env_vars(cmd: &mut Command) {
+    for (key, _) in env::vars() {
+        let should_remove = (key.starts_with("CARGO") && key != "CARGO_HOME" && key != "CARGO_TARGET_DIR")
+            || key.starts_with("RUST")
+            || key == "__CARGO_FIX_YOLO";
+        if should_remove {
+            cmd.env_remove(key);
+        }
+    }
+}
+
+fn publish_module(cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> {
+    run_cli(
+        cli_path,
+        config_path,
+        &[
+            "publish",
+            "--server",
+            server_url,
+            "--module-path",
+            KEYNOTE_MODULE_DIR,
+            "--yes",
+            "--clear-database",
+            DATABASE_NAME,
+        ],
+        "spacetime publish keynote module",
+    )
+}
+
+fn generate_module_bindings(cli_path: &Path, config_path: &Path) -> Result<()> {
+    run_cli(
+        cli_path,
+        config_path,
+        &[
+            "generate",
+            "--lang",
+            "typescript",
+            "--out-dir",
+            KEYNOTE_BINDINGS_DIR,
+            "--module-path",
+            KEYNOTE_MODULE_DIR,
+            "--yes",
+        ],
+        "spacetime generate keynote TypeScript bindings",
+    )
+}
+
+fn seed_accounts(cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> {
+    run_cli(
+        cli_path,
+        config_path,
+        &[
+            "call",
+            "--server",
+            server_url,
+            DATABASE_NAME,
+            "seed",
+            SEED_ACCOUNTS,
+            SEED_INITIAL_BALANCE,
+        ],
+        "spacetime call seed",
+    )
+}
+
+fn run_cli(cli_path: &Path, config_path: &Path, args: &[&str], label: &str) -> Result<()> {
+    let mut cmd = Command::new(cli_path);
+    cmd.arg("--config-path").arg(config_path).args(args);
+    run_command(&mut cmd, label)
+}
+
+fn run_benchmark(server_url: &str, runs_dir: &Path) -> Result<()> {
+    let mut cmd = Command::new("pnpm");
+    cmd.args([
+        "run",
+        "bench",
+        DATABASE_NAME,
+        "--seconds",
+        BENCH_SECONDS,
+        "--concurrency",
+        BENCH_CONCURRENCY,
+        "--connectors",
+        "spacetimedb",
+    ])
+    .current_dir(KEYNOTE_DIR)
+    .env("NODE_ENV", "production")
+    .env("BENCH_PIPELINED", "1")
+    .env("MAX_INFLIGHT_PER_WORKER", MAX_INFLIGHT_PER_WORKER)
+    .env("BENCH_RUNS_DIR", runs_dir)
+    .env("STDB_URL", server_url)
+    .env("STDB_MODULE", DATABASE_NAME)
+    .env("SEED_ACCOUNTS", SEED_ACCOUNTS)
+    .env("SEED_INITIAL_BALANCE", SEED_INITIAL_BALANCE);
+    run_command(&mut cmd, "keynote SpacetimeDB benchmark")
+}
+
+fn run_command(cmd: &mut Command, label: &str) -> Result<()> {
+    let status = cmd.status().with_context(|| format!("failed to spawn {label}"))?;
+    ensure!(status.success(), "{label} failed with status {status}");
+    Ok(())
+}
+
+fn find_result_json(runs_dir: &Path) -> Result<PathBuf> {
+    let mut matches = Vec::new();
+    for entry in fs::read_dir(runs_dir).with_context(|| format!("failed to read {}", runs_dir.display()))? {
+        let entry = entry?;
+        let path = entry.path();
+        let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
+            continue;
+        };
+        if name.starts_with("test-1-spacetimedb-") && name.ends_with(".json") {
+            matches.push(path);
+        }
+    }
+
+    match matches.len() {
+        0 => bail!(
+            "benchmark did not write a test-1-spacetimedb result JSON in {}",
+            runs_dir.display()
+        ),
+        1 => Ok(matches.remove(0)),
+        _ => bail!(
+            "benchmark wrote multiple test-1-spacetimedb result JSON files in {}: {:?}",
+            runs_dir.display(),
+            matches
+        ),
+    }
+}
+
+fn result_tps(result_json: &str) -> Result<f64> {
+    let value: Value = serde_json::from_str(result_json).context("failed to parse benchmark result JSON")?;
+    value
+        .pointer("/results/0/res/tps")
+        .and_then(Value::as_f64)
+        .context("benchmark result JSON is missing results[0].res.tps")
+}

tools/ci/src/main.rs

@@ -13,6 +13,7 @@ use std::{env, fs};
 const README_PATH: &str = "tools/ci/README.md";
 
 mod ci_docs;
+mod keynote_bench;
 mod smoketest;
 mod util;
 
@@ -299,6 +300,11 @@ enum CiCmd {
     ///
     /// Executes the smoketests suite with some default exclusions.
     Smoketests(smoketest::SmoketestsArgs),
+    /// Runs the keynote benchmark as a CI performance regression gate.
+    ///
+    /// Builds release SpacetimeDB binaries, runs the keynote SpacetimeDB benchmark for 60 seconds,
+    /// and fails if throughput is below 250K TPS.
+    KeynoteBench,
     /// Tests the update flow
     ///
     /// Tests the self-update flow by building the spacetimedb-update binary for the specified
@@ -619,6 +625,11 @@ fn main() -> Result<()> {
             smoketest::run(args)?;
         }
 
+        Some(CiCmd::KeynoteBench) => {
+            ensure_repo_root()?;
+            keynote_bench::run()?;
+        }
+
         Some(CiCmd::UpdateFlow {
             target,
             github_token_auth,