diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d516dbc77b..8b72fd5e3dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,6 +265,56 @@ jobs: source ~/emsdk/emsdk_env.sh cargo ci test + keynote_bench: + needs: [lints] + name: Keynote Bench + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 60 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + steps: + - name: Find Git ref + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER="${{ github.event.inputs.pr_number || null }}" + if test -n "${PR_NUMBER}"; then + GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" + else + GIT_REF="${{ github.ref }}" + fi + echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + + - name: Checkout sources + uses: actions/checkout@v4 + with: + ref: ${{ env.GIT_REF }} + + - uses: dsherret/rust-toolchain-file@v1 + - name: Set default rust toolchain + run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: ${{ github.workspace }} + shared-key: spacetimedb + save-if: false + prefix-key: v1 + + # Node 24 is the current Active LTS line. + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 24 + + - uses: ./.github/actions/setup-pnpm + with: + run_install: true + + - name: Run keynote-2 benchmark regression check + run: cargo ci keynote-bench + lints: name: Lints runs-on: spacetimedb-new-runner-2 diff --git a/Cargo.lock b/Cargo.lock index 89dffa918ae..81bdbd49340 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -900,6 +900,7 @@ dependencies = [ "log", "regex", "serde_json", + "spacetimedb-guard", "tempfile", ] diff --git a/templates/keynote-2/src/cli.ts b/templates/keynote-2/src/cli.ts index 0c17b2e56df..6324e767797 100644 --- a/templates/keynote-2/src/cli.ts +++ b/templates/keynote-2/src/cli.ts @@ -5,7 +5,7 @@ import { CONNECTORS } from './connectors'; import { runOne } from './core/runner'; import type { TestCaseModule } from './tests/types'; import { fileURLToPath } from 'node:url'; -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import { RunResult } from './core/types.ts'; import { parseBenchOptions } from './opts.ts'; @@ -153,12 +153,21 @@ function runPrep(): Promise { const testDirUrl = new URL(`./tests/${testName}/`, import.meta.url); const testDirPath = fileURLToPath(testDirUrl); -const runsDir = fileURLToPath(new URL('../runs/', import.meta.url)); +const runsDir = process.env.BENCH_RUNS_DIR + ? resolve(process.env.BENCH_RUNS_DIR) + : fileURLToPath(new URL('../runs/', import.meta.url)); -async function writeRunJson(payload: object, connectorName: string, alpha: number) { +async function writeRunJson( + payload: object, + connectorName: string, + alpha: number, +) { await mkdir(runsDir, { recursive: true }); const ts = new Date().toISOString().replace(/[:.]/g, '-'); - const outFile = join(runsDir, `${testName}-${connectorName}-a${alpha}-${ts}.json`); + const outFile = join( + runsDir, + `${testName}-${connectorName}-a${alpha}-${ts}.json`, + ); await writeFile(outFile, JSON.stringify(payload, null, 2)); console.log(`Wrote results to ${outFile}`); return outFile; diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index e87c89da8ca..79a3b96e24b 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -13,3 +13,4 @@ serde_json.workspace = true duct.workspace = true tempfile.workspace = true env_logger.workspace = true +spacetimedb-guard.workspace = true diff --git a/tools/ci/README.md b/tools/ci/README.md index 9b71b406fef..ddac93a6f27 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -143,6 +143,21 @@ Usage: help [COMMAND]... - `subcommand`: Print help for the subcommand(s) +### `keynote-bench` + +Runs the keynote benchmark as a CI performance regression gate. + +Builds release SpacetimeDB binaries, runs the keynote SpacetimeDB benchmark for 60 seconds, and fails if throughput is below 250K TPS. + +**Usage:** +```bash +Usage: keynote-bench +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + ### `update-flow` Tests the update flow diff --git a/tools/ci/src/keynote_bench.rs b/tools/ci/src/keynote_bench.rs new file mode 100644 index 00000000000..c2b4bcfb72a --- /dev/null +++ b/tools/ci/src/keynote_bench.rs @@ -0,0 +1,216 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{bail, ensure, Context, Result}; +use serde_json::Value; +use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; +use std::{ + env, fs, + path::{Path, PathBuf}, + process::Command, +}; + +const DATABASE_NAME: &str = "test-1"; +const KEYNOTE_DIR: &str = "templates/keynote-2"; +const KEYNOTE_MODULE_DIR: &str = "templates/keynote-2/spacetimedb"; +const KEYNOTE_BINDINGS_DIR: &str = "templates/keynote-2/module_bindings"; +const MIN_TPS: f64 = 250_000.0; +const BENCH_SECONDS: &str = "60"; +const BENCH_CONCURRENCY: &str = "64"; +const MAX_INFLIGHT_PER_WORKER: &str = "64"; +const SEED_ACCOUNTS: &str = "100000"; +const SEED_INITIAL_BALANCE: &str = "1000000000000"; + +pub fn run() -> Result<()> { + build_typescript_sdk()?; + build_release_binaries()?; + + let cli_path = ensure_binaries_built(); + let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let cli_config_dir = tempfile::tempdir().context("failed to create temporary CLI config directory")?; + let cli_config_path = cli_config_dir.path().join("config.toml"); + + publish_module(&cli_path, &cli_config_path, &server.host_url)?; + generate_module_bindings(&cli_path, &cli_config_path)?; + seed_accounts(&cli_path, &cli_config_path, &server.host_url)?; + + let runs_dir = tempfile::tempdir().context("failed to create temporary benchmark runs directory")?; + run_benchmark(&server.host_url, runs_dir.path())?; + + let result_path = find_result_json(runs_dir.path())?; + let result_json = fs::read_to_string(&result_path) + .with_context(|| format!("failed to read benchmark result {}", result_path.display()))?; + let tps = result_tps(&result_json)?; + + if tps < MIN_TPS { + eprintln!( + "Keynote perf regression: throughput {tps:.0} TPS < {MIN_TPS:.0} TPS\n\nResult JSON:\n{}", + result_json + ); + bail!("keynote benchmark throughput is below threshold"); + } + + println!( + "Keynote perf check passed: throughput {tps:.0} TPS >= {MIN_TPS:.0} TPS ({})", + result_path.display() + ); + Ok(()) +} + +fn build_typescript_sdk() -> Result<()> { + let mut cmd = Command::new("pnpm"); + cmd.arg("build").current_dir("crates/bindings-typescript"); + run_command(&mut cmd, "pnpm build in crates/bindings-typescript") +} + +fn build_release_binaries() -> Result<()> { + eprintln!("Building spacetimedb-cli and spacetimedb-standalone (release)..."); + let mut cmd = Command::new("cargo"); + cmd.args([ + "build", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ]); + remove_cargo_env_vars(&mut cmd); + run_command(&mut cmd, "cargo build --release spacetimedb-cli spacetimedb-standalone") +} + +fn remove_cargo_env_vars(cmd: &mut Command) { + for (key, _) in env::vars() { + let should_remove = (key.starts_with("CARGO") && key != "CARGO_HOME" && key != "CARGO_TARGET_DIR") + || key.starts_with("RUST") + || key == "__CARGO_FIX_YOLO"; + if should_remove { + cmd.env_remove(key); + } + } +} + +fn publish_module(cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> { + run_cli( + cli_path, + config_path, + &[ + "publish", + "--server", + server_url, + "--module-path", + KEYNOTE_MODULE_DIR, + "--yes", + "--clear-database", + DATABASE_NAME, + ], + "spacetime publish keynote module", + ) +} + +fn generate_module_bindings(cli_path: &Path, config_path: &Path) -> Result<()> { + run_cli( + cli_path, + config_path, + &[ + "generate", + "--lang", + "typescript", + "--out-dir", + KEYNOTE_BINDINGS_DIR, + "--module-path", + KEYNOTE_MODULE_DIR, + "--yes", + ], + "spacetime generate keynote TypeScript bindings", + ) +} + +fn seed_accounts(cli_path: &Path, config_path: &Path, server_url: &str) -> Result<()> { + run_cli( + cli_path, + config_path, + &[ + "call", + "--server", + server_url, + DATABASE_NAME, + "seed", + SEED_ACCOUNTS, + SEED_INITIAL_BALANCE, + ], + "spacetime call seed", + ) +} + +fn run_cli(cli_path: &Path, config_path: &Path, args: &[&str], label: &str) -> Result<()> { + let mut cmd = Command::new(cli_path); + cmd.arg("--config-path").arg(config_path).args(args); + run_command(&mut cmd, label) +} + +fn run_benchmark(server_url: &str, runs_dir: &Path) -> Result<()> { + let mut cmd = Command::new("pnpm"); + cmd.args([ + "run", + "bench", + DATABASE_NAME, + "--seconds", + BENCH_SECONDS, + "--concurrency", + BENCH_CONCURRENCY, + "--connectors", + "spacetimedb", + ]) + .current_dir(KEYNOTE_DIR) + .env("NODE_ENV", "production") + .env("BENCH_PIPELINED", "1") + .env("MAX_INFLIGHT_PER_WORKER", MAX_INFLIGHT_PER_WORKER) + .env("BENCH_RUNS_DIR", runs_dir) + .env("STDB_URL", server_url) + .env("STDB_MODULE", DATABASE_NAME) + .env("SEED_ACCOUNTS", SEED_ACCOUNTS) + .env("SEED_INITIAL_BALANCE", SEED_INITIAL_BALANCE); + run_command(&mut cmd, "keynote SpacetimeDB benchmark") +} + +fn run_command(cmd: &mut Command, label: &str) -> Result<()> { + let status = cmd.status().with_context(|| format!("failed to spawn {label}"))?; + ensure!(status.success(), "{label} failed with status {status}"); + Ok(()) +} + +fn find_result_json(runs_dir: &Path) -> Result { + let mut matches = Vec::new(); + for entry in fs::read_dir(runs_dir).with_context(|| format!("failed to read {}", runs_dir.display()))? { + let entry = entry?; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name.starts_with("test-1-spacetimedb-") && name.ends_with(".json") { + matches.push(path); + } + } + + match matches.len() { + 0 => bail!( + "benchmark did not write a test-1-spacetimedb result JSON in {}", + runs_dir.display() + ), + 1 => Ok(matches.remove(0)), + _ => bail!( + "benchmark wrote multiple test-1-spacetimedb result JSON files in {}: {:?}", + runs_dir.display(), + matches + ), + } +} + +fn result_tps(result_json: &str) -> Result { + let value: Value = serde_json::from_str(result_json).context("failed to parse benchmark result JSON")?; + value + .pointer("/results/0/res/tps") + .and_then(Value::as_f64) + .context("benchmark result JSON is missing results[0].res.tps") +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index e1bb914e3ff..fcbd705be61 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -13,6 +13,7 @@ use std::{env, fs}; const README_PATH: &str = "tools/ci/README.md"; mod ci_docs; +mod keynote_bench; mod smoketest; mod util; @@ -299,6 +300,11 @@ enum CiCmd { /// /// Executes the smoketests suite with some default exclusions. Smoketests(smoketest::SmoketestsArgs), + /// Runs the keynote benchmark as a CI performance regression gate. + /// + /// Builds release SpacetimeDB binaries, runs the keynote SpacetimeDB benchmark for 60 seconds, + /// and fails if throughput is below 250K TPS. + KeynoteBench, /// Tests the update flow /// /// Tests the self-update flow by building the spacetimedb-update binary for the specified @@ -619,6 +625,11 @@ fn main() -> Result<()> { smoketest::run(args)?; } + Some(CiCmd::KeynoteBench) => { + ensure_repo_root()?; + keynote_bench::run()?; + } + Some(CiCmd::UpdateFlow { target, github_token_auth,