diff --git a/templates/keynote-2/.env.example b/templates/keynote-2/.env.example index a3b0a7aad06..6a247f61f91 100644 --- a/templates/keynote-2/.env.example +++ b/templates/keynote-2/.env.example @@ -1,10 +1,10 @@ # ===== Runtime toggles ===== -USE_DOCKER=1 # 1 = run docker compose up for pg/crdb; 0 = skip +#USE_DOCKER=1 # 1 = run docker compose up for pg/crdb; 0 = skip #SKIP_PG=1 # 1 = don't init Postgres in prep #SKIP_CRDB=1 # 1 = don't init Cockroach in prep #SKIP_SQLITE=1 # 1 = don't init SQLite in prep #SKIP_SUPABASE=1 # 1 = don't init Supabase in prep -SKIP_CONVEX=1 # 1 = don't init Convex in prep +#SKIP_CONVEX=1 # 1 = don't init Convex in prep USE_SPACETIME_METRICS_ENDPOINT=0 # ===== PostgreSQL ===== @@ -39,7 +39,6 @@ SUPABASE_DB_URL=postgresql://postgres:postgres@127.0.0.1:54322/postgres CONVEX_URL=http://127.0.0.1:3210 CONVEX_SITE_URL=http://127.0.0.1:3210 CLEAR_CONVEX_ON_PREP=0 -CONVEX_USE_SHARDED_COUNTER=1 # ===== Bun ===== BUN_URL=http://127.0.0.1:4001 diff --git a/templates/keynote-2/DEVELOP.md b/templates/keynote-2/DEVELOP.md index 1bb90177783..dc8e3ddc929 100644 --- a/templates/keynote-2/DEVELOP.md +++ b/templates/keynote-2/DEVELOP.md @@ -15,7 +15,7 @@ Each run: Run a quick performance comparison: ```bash -npm run demo +pnpm run demo ``` The script will: @@ -40,6 +40,7 @@ The script will: - **Node.js** ≥ 22.x - **pnpm** installed globally +- **Rust** (required for SpacetimeDB benchmarks) -- [install](https://rust-lang.org/tools/install/) - **Docker** for local Postgres / Cockroach / Supabase - Local/Cloud Convex @@ -155,34 +156,34 @@ cd .. 1. Start SpacetimeDB (`cargo run -p spacetimedb-cli -- start` or `spacetime start`) 2. Start Convex (inside convex-app run `npx convex dev`) 3. Init Supabase (run `supabase init`) inside project root. -4. `npm run prep` to seed the databases. -5. `npm run bench` to run the test against all connectors. +4. `pnpm run prep` to seed the databases. +5. `pnpm run bench` to run the test against all connectors. ## Commands & Examples ### 1. Run a test ```bash -npm run bench [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] +pnpm run bench [test-name] [--seconds N] [--concurrency N] [--alpha A] [--connectors list] ``` Examples: ```bash # Default test (test-1), default args (note: only 1 test right now, and it's embedded) -npm run bench +pnpm run bench # Explicit test name -npm run bench test-1 +pnpm run bench test-1 # Short run, 100 concurrent workers -npm run bench test-1 --seconds 10 --concurrency 100 +pnpm run bench test-1 --seconds 10 --concurrency 100 # Heavier skew on hot accounts -npm run bench test-1 --alpha 2.0 +pnpm run bench test-1 --alpha 2.0 # Only run selected connectors -npm run bench test-1 --connectors spacetimedb,sqlite +pnpm run bench test-1 --connectors spacetimedb,sqlite_rpc ``` --- @@ -197,11 +198,11 @@ From `src/cli.ts`: - **`--seconds N`** - Duration of the benchmark in seconds - - Default: `1` + - Default: `10` - **`--concurrency N`** - Number of workers / in-flight operations - - Default: `10` + - Default: `50` - **`--alpha A`** - Zipf α parameter for account selection (hot vs cold distribution) @@ -212,11 +213,12 @@ From `src/cli.ts`: - Example: ```bash - --connectors spacetimedb,sqlite,postgres + --connectors spacetimedb,sqlite_rpc,postgres_rpc ``` - If omitted, all connectors for that test are run - The valid names come from `tc.system` in the test modules and the keys in `CONNECTORS` + - Valid names: `convex`, `spacetimedb`, `bun`, `postgres_rpc`, `cockroach_rpc`, `sqlite_rpc`, `supabase_rpc`, `planetscale_pg_rpc` - **`--contention-tests startAlpha endAlpha step concurrency`** - Runs a sweep over Zipf α values for a single connector @@ -235,14 +237,10 @@ From `src/cli.ts`: You can also run the benchmark via Docker instead of Node directly: ```bash -docker compose run --rm bench \ - --seconds 5 \ - --concurrency 50 \ - --alpha 1 \ - --connectors convex +docker compose run --rm bench -- --seconds 5 --concurrency 50 --alpha 1 --connectors convex ``` -If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `npm prep` before running bench. +If using Docker, make sure to set `USE_DOCKER=1` in `.env`, verify docker-compose env variables, verify you've run supabase init, and run `pnpm run prep` before running bench. ## Output diff --git a/templates/keynote-2/README.md b/templates/keynote-2/README.md index e65a0d1866d..b6bfb29a070 100644 --- a/templates/keynote-2/README.md +++ b/templates/keynote-2/README.md @@ -7,13 +7,13 @@ A benchmark suite comparing SpacetimeDB against traditional web application stac See SpacetimeDB's performance advantage with one command: ```bash -npm install -npm run demo +pnpm install +pnpm run demo ``` The demo compares SpacetimeDB and Convex by default, since both are easy for anyone to set up and run locally without additional infrastructure. Other systems (Postgres, CockroachDB, SQLite, etc.) are also supported but require more setup. The demo checks that required services are running (prompts you to start them if not), seeds databases, and displays animated results. -**Options:** `--systems a,b,c` | `--seconds N` | `--skip-prep` | `--no-animation` +**Options:** `--systems a,b,c` | `--seconds N` | `--concurrency N` | `--alpha N` | `--skip-prep` | `--no-animation` **Note:** You will need to [install Rust](https://rust-lang.org/tools/install/) to run the spacetimedb benchmark, because we run a [Rust Client](#rust-client). @@ -83,7 +83,7 @@ This is a classic read-modify-write workload that tests transactional integrity ### Test Command ```bash -docker compose run --rm bench --seconds 10 --concurrency 50 --alpha XX --connectors YY +docker compose run --rm bench -- --seconds 10 --concurrency 50 --alpha XX --connectors YY ``` - `--seconds 10`: Duration of benchmark run @@ -95,7 +95,7 @@ docker compose run --rm bench --seconds 10 --concurrency 50 --alpha XX --connect **Server Machine (Variant A - PhoenixNAP):** -- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory OS: Ubuntu 24.04 +- s3.c3.medium bare metal instance - Intel i9-14900k 24 cores (32 threads), 128GB DDR5 Memory, OS: Ubuntu 24.04 **Server Machine (Variant B - Google Cloud):** @@ -157,11 +157,11 @@ SpacetimeDB supports `withConfirmedReads` mode which ensures transactions are du PlanetScale results (~477 TPS) demonstrate the **significant impact of cloud database latency**. When the database is accessed over the network (even within the same cloud region), round-trip latency dominates performance. This is why SpacetimeDB's colocated architecture provides such dramatic improvements. -### Rust client +### Rust Client -When running the benchmark for SpacetimeDB on higher-end hardware we found out that we were actually bottlnecked +When running the benchmark for SpacetimeDB on higher-end hardware we found out that we were actually bottlenecked on our test TypeScript client. To get the absolute most out of the performance of SpacetimeDB we wrote a custom -Rust client that allows us to send a much larger number of requests then we could otherwise. We didn't do this +Rust client that allows us to send a much larger number of requests than we could otherwise. We didn't do this for the other backends/databases as they maxed out before the client. ## Systems Tested @@ -179,26 +179,7 @@ for the other backends/databases as they maxed out before the client. ## Running the Benchmarks -See [DEVELOP.md](./DEVELOP.md) for detailed setup and execution instructions. - -### Quick Start - -```bash -# Install dependencies -pnpm install - -# Copy environment config -cp .env.example .env - -# Start services (PostgreSQL, CockroachDB, etc.) -docker compose up -d pg crdb - -# Seed databases -npm run prep - -# Run benchmark -npm run bench -- --seconds 10 --concurrency 50 --alpha 1.5 --connectors spacetimedb,postgres_rpc,sqlite_rpc -``` +See [DEVELOP.md](./DEVELOP.md) for prerequisites, configuration, and full CLI reference. ## Output diff --git a/templates/keynote-2/src/cli.ts b/templates/keynote-2/src/cli.ts index 0ce7ea5721b..cf265215ce6 100644 --- a/templates/keynote-2/src/cli.ts +++ b/templates/keynote-2/src/cli.ts @@ -1,4 +1,4 @@ -import 'dotenv/config'; +import 'dotenv/config'; import { readdir, mkdir, writeFile } from 'node:fs/promises'; import { CONNECTORS } from './connectors'; import { runOne } from './core/runner'; @@ -17,8 +17,8 @@ if (args.length > 0 && !args[0].startsWith('--')) { posArgs = args.slice(1); } -let seconds = 1, - concurrency = 10, +let seconds = 10, + concurrency = 50, accounts = process.env.SEED_ACCOUNTS ? Number(process.env.SEED_ACCOUNTS) : 100_000, diff --git a/templates/keynote-2/src/demo.ts b/templates/keynote-2/src/demo.ts index 78dbc806d62..2c8b17147af 100644 --- a/templates/keynote-2/src/demo.ts +++ b/templates/keynote-2/src/demo.ts @@ -70,7 +70,7 @@ function hasFlag(name: string): boolean { } const seconds = getArg('seconds', 10); -const concurrency = getArg('concurrency', 10); +const concurrency = getArg('concurrency', 50); const alpha = getArg('alpha', 1.5); const systems = getStringArg('systems', 'convex,spacetimedb') .split(',')