From 5d03b352cdaabd192d9e7e4e69df0b6ff15d3f53 Mon Sep 17 00:00:00 2001 From: Clay Smith Date: Wed, 17 Jun 2026 13:51:49 -0700 Subject: [PATCH] add cost walkthrough video pipeline --- .github/workflows/demo-videos.yml | 44 + .gitignore | 9 + demo-videos/README.md | 157 ++ demo-videos/capture/record.ts | 251 ++ demo-videos/capture/selectors.ts | 8 + demo-videos/capture/types.ts | 341 +++ demo-videos/output/.gitkeep | 1 + demo-videos/remotion/src/Root.tsx | 22 + demo-videos/remotion/src/Walkthrough.tsx | 200 ++ .../remotion/src/components/BrandBug.tsx | 22 + .../remotion/src/components/BrowserVideo.tsx | 19 + .../remotion/src/components/Callouts.tsx | 54 + .../remotion/src/components/Captions.tsx | 27 + .../remotion/src/components/TitleCard.tsx | 25 + .../walkthroughs/cost-walkthrough.yaml | 117 + introduction/how-tero-works.mdx | 7 + package.json | 24 +- pnpm-lock.yaml | 2435 ++++++++++++++--- public/videos/.gitkeep | 1 + public/videos/cost-walkthrough-poster.png | Bin 0 -> 91960 bytes public/videos/cost-walkthrough.mp4 | Bin 0 -> 10196250 bytes scripts/demo-video/chatterbox_voiceover.py | 132 + scripts/demo-video/codex-qa.ts | 95 + scripts/demo-video/copy-assets.ts | 50 + scripts/demo-video/extract-narration.ts | 42 + scripts/demo-video/generate-captions.ts | 78 + scripts/demo-video/generate-voiceover.ts | 392 +++ scripts/demo-video/install-chatterbox.sh | 33 + scripts/demo-video/render.ts | 89 + tsconfig.json | 16 + 30 files changed, 4367 insertions(+), 324 deletions(-) create mode 100644 .github/workflows/demo-videos.yml create mode 100644 demo-videos/README.md create mode 100644 demo-videos/capture/record.ts create mode 100644 demo-videos/capture/selectors.ts create mode 100644 demo-videos/capture/types.ts create mode 100644 demo-videos/output/.gitkeep create mode 100644 demo-videos/remotion/src/Root.tsx create mode 100644 demo-videos/remotion/src/Walkthrough.tsx create mode 100644 demo-videos/remotion/src/components/BrandBug.tsx create mode 100644 demo-videos/remotion/src/components/BrowserVideo.tsx create mode 100644 demo-videos/remotion/src/components/Callouts.tsx create mode 100644 demo-videos/remotion/src/components/Captions.tsx create mode 100644 demo-videos/remotion/src/components/TitleCard.tsx create mode 100644 demo-videos/walkthroughs/cost-walkthrough.yaml create mode 100644 public/videos/.gitkeep create mode 100644 public/videos/cost-walkthrough-poster.png create mode 100644 public/videos/cost-walkthrough.mp4 create mode 100755 scripts/demo-video/chatterbox_voiceover.py create mode 100644 scripts/demo-video/codex-qa.ts create mode 100644 scripts/demo-video/copy-assets.ts create mode 100644 scripts/demo-video/extract-narration.ts create mode 100644 scripts/demo-video/generate-captions.ts create mode 100644 scripts/demo-video/generate-voiceover.ts create mode 100755 scripts/demo-video/install-chatterbox.sh create mode 100644 scripts/demo-video/render.ts create mode 100644 tsconfig.json diff --git a/.github/workflows/demo-videos.yml b/.github/workflows/demo-videos.yml new file mode 100644 index 0000000..9c2a1d3 --- /dev/null +++ b/.github/workflows/demo-videos.yml @@ -0,0 +1,44 @@ +name: Demo videos + +on: + workflow_dispatch: + inputs: + walkthrough: + description: Walkthrough id + default: cost-walkthrough + required: true + pull_request: + paths: + - "demo-videos/**" + - "scripts/demo-video/**" + - "package.json" + - "pnpm-lock.yaml" + - ".github/workflows/demo-videos.yml" + +jobs: + render: + runs-on: ubuntu-latest + env: + DEMO_BASE_URL: ${{ vars.DEMO_BASE_URL || 'https://demo.usetero.com' }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10.12.4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install --with-deps chromium + - run: pnpm video:extract -- ${{ inputs.walkthrough || 'cost-walkthrough' }} + - run: pnpm video:capture -- ${{ inputs.walkthrough || 'cost-walkthrough' }} + - run: pnpm video:voice -- ${{ inputs.walkthrough || 'cost-walkthrough' }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + - run: pnpm video:captions -- ${{ inputs.walkthrough || 'cost-walkthrough' }} + - run: pnpm video:render -- ${{ inputs.walkthrough || 'cost-walkthrough' }} + - uses: actions/upload-artifact@v4 + with: + name: demo-videos + path: public/videos diff --git a/.gitignore b/.gitignore index 0b54752..083d562 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,15 @@ node_modules/ .mintlify/ _dist/ +# Demo video outputs +demo-videos/.venv-chatterbox/ +demo-videos/output/* +!demo-videos/output/.gitkeep +public/videos/* +!public/videos/.gitkeep +!public/videos/*.mp4 +!public/videos/*-poster.png + # macOS .DS_Store diff --git a/demo-videos/README.md b/demo-videos/README.md new file mode 100644 index 0000000..e4d745d --- /dev/null +++ b/demo-videos/README.md @@ -0,0 +1,157 @@ +# Demo Video Pipeline + +This directory owns deterministic product walkthrough video generation for the docs. + +The production path is scripted and CI-friendly: + +1. Read a walkthrough YAML file. +2. Capture browser footage with Playwright and stable `data-demo-id` selectors. +3. Extract narration, generate or stub voiceover audio, and create captions. +4. Stage artifacts into `public/videos//`. +5. Render an MP4 and poster with Remotion. + +AI/browser agents can help author or QA walkthroughs, but the render path must not rely on a live agent clicking through the UI. + +## Codex-local workflow + +Codex can run this locally with your existing Codex CLI sign-in: + +```bash +codex login +pnpm video:cost-walkthrough +pnpm video:codex:qa -- cost-walkthrough +``` + +Use Codex for authoring and QA, not for production clicking. The `video:codex:qa` command runs `codex exec` over the walkthrough YAML and generated artifacts, then writes `demo-videos/output//codex-qa.md`. + +Your Codex CLI or ChatGPT subscription does not act as a general OpenAI API key for scripts. OpenAI speech generation still requires `OPENAI_API_KEY`. On macOS, the voice step automatically uses the local `say` tool when no API key is present. You can also request it explicitly: + +```bash +DEMO_VIDEO_TTS_PROVIDER=macos pnpm video:voice -- cost-walkthrough +``` + +Without `OPENAI_API_KEY` or an available local macOS speech tool, the voice step writes deterministic placeholder silence. + +## Walkthrough YAML + +Walkthroughs live in `demo-videos/walkthroughs/`. Each file defines the demo URL, viewport, voice settings, and ordered scenes. Browser scenes support these deterministic steps: + +- `waitFor`: wait for a selector to be visible. +- `click`: click a selector. +- `spotlight`: record a selector bounding box for a callout overlay. +- `pause`: wait a fixed number of seconds. +- `fill`: fill a selector with text. +- `press`: press a keyboard key on a selector. + +The canonical walkthrough is `demo-videos/walkthroughs/cost-walkthrough.yaml`. + +## Demo Selectors + +The demo site should prefer stable `data-demo-id` attributes for new flows. The current `cost-walkthrough` walkthrough starts with existing production DOM on the hosted demo and uses these selectors: + +- `a[href="/cost"][aria-label^="Cost,"]` +- `a[href="/issues?statusTab=open&view=cost&severity=high"]` +- `section[aria-label="Issues"]` +- `button[aria-label^="ISS-1:"]` +- `button[aria-label="Deploy policy from issue footer"]` + +The hosted demo starts at `https://demo.usetero.com/`, then the walkthrough clicks through the app navigation to `/cost` before opening the filtered cost issue queue. If this flow becomes permanent, add explicit `data-demo-id` attributes in the app and update the YAML to use those instead of ARIA/text-derived selectors. + +## Local Commands + +Install dependencies and Playwright Chromium: + +```bash +pnpm install +pnpm exec playwright install chromium +``` + +Run the full `cost-walkthrough` pipeline: + +```bash +DEMO_BASE_URL=http://localhost:5173 pnpm video:cost-walkthrough +``` + +Use the hosted demo instead: + +```bash +DEMO_BASE_URL=https://demo.usetero.com pnpm video:cost-walkthrough +``` + +Run individual steps: + +```bash +pnpm video:extract -- cost-walkthrough +pnpm video:capture -- cost-walkthrough +pnpm video:voice -- cost-walkthrough +pnpm video:captions -- cost-walkthrough +pnpm video:render -- cost-walkthrough +``` + +Outputs are written to `demo-videos/output//`. Rendered assets are staged under `public/videos//`, with final docs assets at: + +- `public/videos/.mp4` +- `public/videos/-poster.png` + +Intermediate capture and render assets are ignored by git by default. Commit the final MP4 and poster when a docs page embeds them. + +## Voiceover + +Set `OPENAI_API_KEY` to generate narrated audio with the OpenAI speech API: + +```bash +OPENAI_API_KEY=... pnpm video:voice -- cost-walkthrough +``` + +Without `OPENAI_API_KEY`, the voice step writes deterministic placeholder silence and a note explaining what is missing. This keeps local and CI runs resilient when secrets are unavailable. + +On macOS, the script uses local system speech to create `voiceover.wav` when no `OPENAI_API_KEY` is present. Set `DEMO_VIDEO_TTS_PROVIDER=macos` to force that path, or `DEMO_VIDEO_TTS_PROVIDER=placeholder` to force deterministic placeholder silence. If local speech is blocked or unavailable, the script falls back to placeholder silence. + +### Local Chatterbox TTS + +Install Resemble AI Chatterbox into a repo-local `uv` environment: + +```bash +pnpm video:install-chatterbox +``` + +Generate scene voiceover with the local model: + +```bash +DEMO_VIDEO_TTS_PROVIDER=chatterbox pnpm video:voice -- cost-walkthrough +``` + +The provider writes per-scene `voiceover-.wav` files and a `voiceover.json` manifest, so capture timing and Remotion rendering work the same way as the OpenAI and macOS providers. On macOS, the provider uses the `Samantha` system voice as a local reference prompt by default so Chatterbox produces a more professional female voice. + +Optional Chatterbox settings: + +- `DEMO_VIDEO_CHATTERBOX_MODEL=standard|turbo|multilingual` +- `DEMO_VIDEO_CHATTERBOX_DEVICE=auto|cuda|mps|cpu` +- `DEMO_VIDEO_CHATTERBOX_AUDIO_PROMPT=/path/to/reference.wav` +- `DEMO_VIDEO_CHATTERBOX_MACOS_VOICE=Samantha` +- `DEMO_VIDEO_CHATTERBOX_DISABLE_MACOS_PROMPT=1` +- `DEMO_VIDEO_CHATTERBOX_LANGUAGE_ID=en` +- `DEMO_VIDEO_CHATTERBOX_EXAGGERATION=0.5` +- `DEMO_VIDEO_CHATTERBOX_CFG_WEIGHT=0.5` +- `DEMO_VIDEO_CHATTERBOX_TEMPERATURE=0.8` + +## Adding a Walkthrough + +1. Copy `demo-videos/walkthroughs/cost-walkthrough.yaml`. +2. Give it a unique `id`, globally clear `title`, and scene IDs. +3. Add stable `data-demo-id` attributes to the demo app for every action and callout. +4. Run `pnpm video:extract`, `pnpm video:capture`, `pnpm video:voice`, `pnpm video:captions`, and `pnpm video:render`. +5. Embed the final MP4 in the relevant MDX page: + +```mdx +