From 41dbe6a0a6a04a6224fec681eb42772264ff91ac Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Mon, 13 Apr 2026 16:04:34 -0700 Subject: [PATCH 01/11] Use static FlexSearch for docs search --- bun.lock | 3 + package.json | 6 +- scripts/generate-search-index.ts | 59 +++ scripts/search-benchmark.ts | 657 +++++++++++++++++++++++++++++++ src/components/SearchDialog.tsx | 19 +- src/lib/search-index.ts | 126 ++++++ src/lib/search.shared.ts | 4 + src/lib/search.ts | 128 ++++++ src/lib/static-search-client.ts | 50 +++ src/routes/api/search.ts | 11 +- 10 files changed, 1051 insertions(+), 12 deletions(-) create mode 100644 scripts/generate-search-index.ts create mode 100644 scripts/search-benchmark.ts create mode 100644 src/lib/search-index.ts create mode 100644 src/lib/search.shared.ts create mode 100644 src/lib/search.ts create mode 100644 src/lib/static-search-client.ts diff --git a/bun.lock b/bun.lock index 6045910..0b92017 100644 --- a/bun.lock +++ b/bun.lock @@ -14,6 +14,7 @@ "@tanstack/react-start": "1.166.1", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", + "flexsearch": "^0.8.212", "fumadocs-core": "16.7.10", "fumadocs-mdx": "14.2.11", "fumadocs-ui": "16.7.10", @@ -938,6 +939,8 @@ "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + "flexsearch": ["flexsearch@0.8.212", "", {}, "sha512-wSyJr1GUWoOOIISRu+X2IXiOcVfg9qqBRyCPRUdLMIGJqPzMo+jMRlvE83t14v1j0dRMEaBbER/adQjp6Du2pw=="], + "format": ["format@0.2.2", "", {}, "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww=="], "framer-motion": ["framer-motion@12.38.0", "", { "dependencies": { "motion-dom": "^12.38.0", "motion-utils": "^12.36.0", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-rFYkY/pigbcswl1XQSb7q424kSTQ8q6eAC+YUsSKooHQYuLdzdHjrt6uxUC+PRAO++q5IS7+TamgIw1AphxR+g=="], diff --git a/package.json b/package.json index bb447a6..7829fbb 100644 --- a/package.json +++ b/package.json @@ -12,12 +12,13 @@ "dev": "vite dev", "dev:port": "vite dev --port", "prebuild": "bun run generate:changelog && bun run scripts/copy-docs-images.cjs", - "build": "NODE_OPTIONS=--max-old-space-size=8192 vite build && bun run scripts/generate-static-cache.ts", + "build": "NODE_OPTIONS=--max-old-space-size=8192 vite build && bun run scripts/generate-static-cache.ts && bun run scripts/generate-search-index.ts", "build:cf": "bun run build", - "build:cf:staging": "NODE_OPTIONS=--max-old-space-size=8192 CLOUDFLARE_ENV=staging vite build", + "build:cf:staging": "NODE_OPTIONS=--max-old-space-size=8192 CLOUDFLARE_ENV=staging bun run build", "sync:mixedbread": "mxbai vs sync $MIXEDBREAD_STORE_ID './content/docs' --ci", "preview": "vite preview", "preview:cf": "bun run build:cf && vite preview", + "benchmark:search": "bun run scripts/search-benchmark.ts", "deploy": "bun run build:cf && wrangler deploy", "deploy:staging": "bun run build:cf:staging && wrangler deploy", "cf:typegen": "wrangler types", @@ -38,6 +39,7 @@ "@tanstack/react-start": "1.166.1", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", + "flexsearch": "^0.8.212", "fumadocs-core": "16.7.10", "fumadocs-mdx": "14.2.11", "fumadocs-ui": "16.7.10", diff --git a/scripts/generate-search-index.ts b/scripts/generate-search-index.ts new file mode 100644 index 0000000..caa9c69 --- /dev/null +++ b/scripts/generate-search-index.ts @@ -0,0 +1,59 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { createServer } from "vite"; +import react from "@vitejs/plugin-react"; +import mdx from "fumadocs-mdx/vite"; +import tsConfigPaths from "vite-tsconfig-paths"; + +const DIST_CLIENT = path.join(process.cwd(), "dist/client"); + +async function main() { + console.log("Generating static search index…"); + + const server = await createServer({ + configFile: false, + logLevel: "error", + server: { port: 0, host: "127.0.0.1" }, + resolve: { + alias: { "@": path.resolve(process.cwd(), "./src") }, + }, + plugins: [ + mdx(await import("../source.config")), + tsConfigPaths({ projects: ["./tsconfig.json"] }), + react(), + ], + }); + + try { + const { buildDocsSearchDocuments } = await server.ssrLoadModule( + "./src/lib/search", + ); + const { SEARCH_INDEX_FILENAME } = await server.ssrLoadModule( + "./src/lib/search.shared", + ); + const { buildStaticSearchIndex, exportStaticSearchIndex } = await server.ssrLoadModule( + "./src/lib/search-index", + ); + + const outputDir = path.join(DIST_CLIENT, "docs"); + const outputPath = path.join(outputDir, SEARCH_INDEX_FILENAME); + const documents = await buildDocsSearchDocuments(); + const index = buildStaticSearchIndex(documents); + const payload = JSON.stringify(exportStaticSearchIndex(index)); + + await fs.mkdir(outputDir, { recursive: true }); + await fs.writeFile(outputPath, payload); + + const bytes = Buffer.byteLength(payload); + console.log( + ` ✓ search index written to ${path.relative(process.cwd(), outputPath)} (${bytes.toLocaleString()} bytes)`, + ); + } finally { + await server.close(); + } +} + +main().catch((err) => { + console.error("Static search index generation failed:", err); + process.exit(1); +}); diff --git a/scripts/search-benchmark.ts b/scripts/search-benchmark.ts new file mode 100644 index 0000000..f1faffa --- /dev/null +++ b/scripts/search-benchmark.ts @@ -0,0 +1,657 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +type QueryCase = { + name: string; + query: string; + expectedText?: string; +}; + +type SearchRequestRecord = { + requestId: string; + url: string; + query: string | null; + kind: "api" | "index"; + method: string; + status?: number; + observedAtMs: number; +}; + +type RawQueryResult = { + name: string; + query: string; + expectedText?: string; + inputSetMs: number; + firstTransportRequestSeenMs: number | null; + firstVisibleResultMs: number | null; + transportRequestCount: number; + transportRequests: Array<{ + kind: "api" | "index"; + query: string | null; + url: string; + status?: number; + seenOffsetMs: number | null; + }>; +}; + +type TypingSimulationResult = { + query: string; + cadenceMs: number; + totalWallMs: number; + firstTransportRequestSeenMs: number | null; + finalTransportRequestSeenMs: number | null; + finalResultVisibleMs: number | null; + transportRequests: Array<{ + kind: "api" | "index"; + query: string | null; + url: string; + status?: number; + seenOffsetMs: number | null; + }>; +}; + +type BenchmarkResult = { + url: string; + generatedAt: string; + outputPath: string; + sessionName: string; + userAgent: string; + viewport: { + width: number; + height: number; + deviceScaleFactor: number; + }; + navigation: { + openMs: number; + searchButtonReadyMs: number; + metrics: Record | null; + }; + searchDialog: { + openMs: number; + }; + rawQueries: RawQueryResult[]; + typingSimulation: TypingSimulationResult; +}; + +type CliOptions = { + url: string; + outputPath?: string; + queriesPath?: string; + cadenceMs: number; + timeoutMs: number; + width: number; + height: number; + deviceScaleFactor: number; +}; + +function parseArgs(argv: string[]): CliOptions { + const options: CliOptions = { + url: "", + cadenceMs: 100, + timeoutMs: 10_000, + width: 1440, + height: 900, + deviceScaleFactor: 1, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const next = argv[i + 1]; + + switch (arg) { + case "--url": + options.url = next ?? ""; + i += 1; + break; + case "--output": + options.outputPath = next; + i += 1; + break; + case "--queries": + options.queriesPath = next; + i += 1; + break; + case "--typing-cadence-ms": + options.cadenceMs = Number(next); + i += 1; + break; + case "--timeout-ms": + options.timeoutMs = Number(next); + i += 1; + break; + case "--width": + options.width = Number(next); + i += 1; + break; + case "--height": + options.height = Number(next); + i += 1; + break; + case "--device-scale-factor": + options.deviceScaleFactor = Number(next); + i += 1; + break; + default: + break; + } + } + + if (!options.url) { + throw new Error("missing required --url"); + } + + return options; +} + +function sanitizeFileSegment(value: string): string { + return value + .toLowerCase() + .replace(/^https?:\/\//, "") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80); +} + +function defaultQueries(url: URL): QueryCase[] { + if (url.pathname.startsWith("/docs")) { + return [ + { + name: "purchase-controller", + query: "purchase controller", + expectedText: "PurchaseController", + }, + { + name: "apple-search-ads", + query: "apple search ads", + expectedText: "Apple Search Ads", + }, + { + name: "refund-protection", + query: "refund protection", + expectedText: "Refund Protection", + }, + ]; + } + + return [ + { + name: "search", + query: "search", + expectedText: "Search", + }, + ]; +} + +function defaultTypingQuery(url: URL): QueryCase { + if (url.pathname.startsWith("/docs")) { + return { + name: "posthog", + query: "posthog", + expectedText: "PostHog", + }; + } + + return { + name: "typing-search", + query: "search", + expectedText: "Search", + }; +} + +async function loadQueries(options: CliOptions): Promise { + if (!options.queriesPath) { + return defaultQueries(new URL(options.url)); + } + + const file = Bun.file(options.queriesPath); + return (await file.json()) as QueryCase[]; +} + +async function runCommand(command: string[], cwd: string): Promise { + const proc = Bun.spawn({ + cmd: command, + cwd, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + env: process.env, + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + if (exitCode !== 0) { + throw new Error( + `command failed (${exitCode}): ${command.join(" ")}\n${stdout}\n${stderr}`.trim(), + ); + } + + return stdout.trim(); +} + +async function runAgentBrowser( + sessionName: string, + profileDir: string, + cwd: string, + args: string[], +): Promise { + return runCommand( + ["agent-browser", "--session-name", sessionName, "--profile", profileDir, ...args], + cwd, + ); +} + +async function sleep(ms: number): Promise { + await Bun.sleep(ms); +} + +async function waitFor( + fn: () => Promise, + predicate: (value: T) => boolean, + timeoutMs: number, + intervalMs = 50, + label = "condition", +): Promise { + const start = performance.now(); + + while (true) { + const value = await fn(); + if (predicate(value)) return value; + if (performance.now() - start > timeoutMs) { + throw new Error(`timed out waiting for ${label}`); + } + await sleep(intervalMs); + } +} + + +async function dialogIncludesText( + sessionName: string, + profileDir: string, + cwd: string, + text: string, +): Promise { + const snapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot"]); + return snapshot.includes(text); +} + +async function listSearchRequests( + sessionName: string, + profileDir: string, + cwd: string, +): Promise { + const output = await runAgentBrowser(sessionName, profileDir, cwd, [ + "network", + "requests", + "--type", + "fetch", + ]); + const observedAtMs = Date.now(); + const lines = output.split("\n"); + const requests: SearchRequestRecord[] = []; + + for (const line of lines) { + const match = line.match(/^\[([^\]]+)\]\s+([A-Z]+)\s+(https?:\/\/\S+)\s+\(Fetch\)(?:\s+(\d{3}))?$/); + if (!match) continue; + + const [, requestId, method, url, status] = match; + const parsedUrl = new URL(url); + let kind: SearchRequestRecord["kind"] | null = null; + + if (parsedUrl.pathname.endsWith("/api/search")) { + kind = "api"; + } else if (parsedUrl.pathname.endsWith("/search-index.json")) { + kind = "index"; + } + + if (!kind) continue; + + requests.push({ + requestId, + kind, + method, + url, + query: + kind === "api" + ? parsedUrl.searchParams.get("query") ?? parsedUrl.searchParams.get("q") + : null, + status: status ? Number(status) : undefined, + observedAtMs, + }); + } + + return requests; +} + +function startSearchRequestObserver( + sessionName: string, + profileDir: string, + cwd: string, + knownRequestIds: Set, + intervalMs = 100, +) { + let stopped = false; + const observed = new Map(); + + const task = (async () => { + while (!stopped) { + const requests = await listSearchRequests(sessionName, profileDir, cwd); + for (const request of requests) { + if (knownRequestIds.has(request.requestId)) continue; + if (observed.has(request.requestId)) continue; + observed.set(request.requestId, request); + } + + await sleep(intervalMs); + } + + return [...observed.values()].sort((left, right) => left.observedAtMs - right.observedAtMs); + })(); + + return { + async stop() { + stopped = true; + return task; + }, + }; +} + +async function benchmarkRawQuery( + sessionName: string, + profileDir: string, + cwd: string, + inputRef: string, + queryCase: QueryCase, + timeoutMs: number, +): Promise { + await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, ""]); + await sleep(250); + const knownRequestIds = new Set( + (await listSearchRequests(sessionName, profileDir, cwd)).map((request) => request.requestId), + ); + const requestObserver = startSearchRequestObserver(sessionName, profileDir, cwd, knownRequestIds); + const startedAt = Date.now(); + const fillStart = performance.now(); + await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, queryCase.query]); + const inputSetMs = Number((performance.now() - fillStart).toFixed(2)); + + let firstVisibleResultMs: number | null = null; + try { + if (queryCase.expectedText) { + await waitFor( + async () => { + const isVisible = await dialogIncludesText(sessionName, profileDir, cwd, queryCase.expectedText!); + if (isVisible && firstVisibleResultMs === null) { + firstVisibleResultMs = Number((Date.now() - startedAt).toFixed(2)); + } + return isVisible; + }, + Boolean, + timeoutMs, + ); + } else { + await sleep(1_000); + } + } finally { + await sleep(150); + } + + const transportRequests = await requestObserver.stop(); + const firstRequest = transportRequests[0]; + + return { + name: queryCase.name, + query: queryCase.query, + expectedText: queryCase.expectedText, + inputSetMs, + firstTransportRequestSeenMs: + firstRequest ? Number((firstRequest.observedAtMs - startedAt).toFixed(2)) : null, + firstVisibleResultMs, + transportRequestCount: transportRequests.length, + transportRequests: transportRequests.map((request) => ({ + kind: request.kind, + query: request.query, + url: request.url, + status: request.status, + seenOffsetMs: Number((request.observedAtMs - startedAt).toFixed(2)), + })), + }; +} + +async function benchmarkTypingSimulation( + sessionName: string, + profileDir: string, + cwd: string, + inputRef: string, + finalQuery: QueryCase, + cadenceMs: number, + timeoutMs: number, +): Promise { + await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, ""]); + await sleep(250); + const knownRequestIds = new Set( + (await listSearchRequests(sessionName, profileDir, cwd)).map((request) => request.requestId), + ); + const requestObserver = startSearchRequestObserver(sessionName, profileDir, cwd, knownRequestIds); + const startedAt = Date.now(); + let currentValue = ""; + + for (const char of finalQuery.query) { + currentValue = `${currentValue}${char}`; + await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, currentValue]); + if (!currentValue.endsWith(char)) { + throw new Error("failed to update search input during typing simulation"); + } + await sleep(cadenceMs); + } + + let finalResultVisibleMs: number | null = null; + try { + if (finalQuery.expectedText) { + await waitFor( + async () => { + const isVisible = await dialogIncludesText( + sessionName, + profileDir, + cwd, + finalQuery.expectedText!, + ); + if (isVisible && finalResultVisibleMs === null) { + finalResultVisibleMs = Number((Date.now() - startedAt).toFixed(2)); + } + return isVisible; + }, + Boolean, + timeoutMs, + ); + } else { + await sleep(1_000); + } + } finally { + await sleep(150); + } + + const transportRequests = await requestObserver.stop(); + + const requests = transportRequests.map((request) => ({ + kind: request.kind, + query: request.query, + url: request.url, + status: request.status, + seenOffsetMs: Number((request.observedAtMs - startedAt).toFixed(2)), + })); + + const requestSeenOffsets = requests + .map((request) => request.seenOffsetMs) + .filter((value): value is number => value != null) + .sort((left, right) => left - right); + const totalWallMs = Number((Date.now() - startedAt).toFixed(2)); + + return { + query: finalQuery.query, + cadenceMs, + totalWallMs, + firstTransportRequestSeenMs: requestSeenOffsets[0] ?? null, + finalTransportRequestSeenMs: requestSeenOffsets.at(-1) ?? null, + finalResultVisibleMs, + transportRequests: requests, + }; +} + +function printSummary(result: BenchmarkResult) { + const lines = [ + `URL: ${result.url}`, + `Output: ${result.outputPath}`, + "", + `Initial load: ${result.navigation.openMs.toFixed(2)}ms`, + `Search button ready: ${result.navigation.searchButtonReadyMs.toFixed(2)}ms`, + `Search dialog open: ${result.searchDialog.openMs.toFixed(2)}ms`, + "", + "Raw queries:", + ]; + + for (const item of result.rawQueries) { + lines.push( + `- ${item.name}: visible=${item.firstVisibleResultMs ?? "n/a"}ms, transport=${item.transportRequestCount}`, + ); + } + + lines.push(""); + lines.push( + `Typing simulation (${result.typingSimulation.query}, cadence ${result.typingSimulation.cadenceMs}ms): visible=${result.typingSimulation.finalResultVisibleMs ?? "n/a"}ms, transport=${result.typingSimulation.transportRequests.length}`, + ); + + console.log(lines.join("\n")); +} + +function parseSnapshotRef(snapshot: string, role: "button" | "textbox", labelPattern: RegExp): string { + const lines = snapshot.split("\n"); + for (const line of lines) { + if (!line.includes(`- ${role} `)) continue; + if (!labelPattern.test(line)) continue; + const match = line.match(/\[ref=(e\d+)\]/); + if (match) return `@${match[1]}`; + } + + throw new Error(`could not find ${role} ref matching ${labelPattern}`); +} + +async function main() { + const cwd = process.cwd(); + const options = parseArgs(process.argv.slice(2)); + const url = new URL(options.url); + const queries = await loadQueries(options); + const typingQuery = defaultTypingQuery(url); + + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const sessionName = `search-benchmark-${Date.now().toString(36)}`; + const profileDir = path.join(cwd, ".context", "browser-profiles", sessionName); + const outputPath = + options.outputPath ?? + path.join( + cwd, + ".context", + `search-benchmark-${sanitizeFileSegment(`${url.host}${url.pathname}`)}-${timestamp}.json`, + ); + + await mkdir(path.dirname(outputPath), { recursive: true }); + await mkdir(profileDir, { recursive: true }); + + await runAgentBrowser(sessionName, profileDir, cwd, [ + "open", + "about:blank", + ]); + await runAgentBrowser(sessionName, profileDir, cwd, [ + "set", + "viewport", + String(options.width), + String(options.height), + String(options.deviceScaleFactor), + ]); + + try { + const navigationStart = performance.now(); + await runAgentBrowser(sessionName, profileDir, cwd, ["open", options.url]); + const openMs = Number((performance.now() - navigationStart).toFixed(2)); + + const metrics = null; + const userAgent = "agent-browser"; + const searchButtonReadyMs = openMs; + + const snapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot", "-i"]); + const searchButtonRef = parseSnapshotRef(snapshot, "button", /Search/i); + + const openDialogStart = performance.now(); + await runAgentBrowser(sessionName, profileDir, cwd, ["click", searchButtonRef]); + await waitFor( + async () => { + const latestSnapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot", "-i"]); + return latestSnapshot.includes('textbox "Search"'); + }, + Boolean, + options.timeoutMs, + 50, + "search dialog input", + ); + const searchDialogOpenMs = Number((performance.now() - openDialogStart).toFixed(2)); + const dialogSnapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot", "-i"]); + const inputRef = parseSnapshotRef(dialogSnapshot, "textbox", /^- textbox "Search"/); + + const rawQueries: RawQueryResult[] = []; + for (const queryCase of queries) { + rawQueries.push( + await benchmarkRawQuery( + sessionName, + profileDir, + cwd, + inputRef, + queryCase, + options.timeoutMs, + ), + ); + } + + const typingSimulation = await benchmarkTypingSimulation( + sessionName, + profileDir, + cwd, + inputRef, + typingQuery, + options.cadenceMs, + options.timeoutMs, + ); + + const result: BenchmarkResult = { + url: options.url, + generatedAt: new Date().toISOString(), + outputPath, + sessionName, + userAgent, + viewport: { + width: options.width, + height: options.height, + deviceScaleFactor: options.deviceScaleFactor, + }, + navigation: { + openMs, + searchButtonReadyMs, + metrics, + }, + searchDialog: { + openMs: searchDialogOpenMs, + }, + rawQueries, + typingSimulation, + }; + + await writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`, "utf8"); + printSummary(result); + } finally { + await runAgentBrowser(sessionName, profileDir, cwd, ["close"]).catch(() => undefined); + } +} + +await main(); diff --git a/src/components/SearchDialog.tsx b/src/components/SearchDialog.tsx index ee214a4..94677ba 100644 --- a/src/components/SearchDialog.tsx +++ b/src/components/SearchDialog.tsx @@ -15,7 +15,10 @@ import { TagsListItem, } from "fumadocs-ui/components/dialog/search"; import { useDocsSearch } from "fumadocs-core/search/client"; +import { fetchClient } from "fumadocs-core/search/client/fetch"; import type { SharedProps } from "fumadocs-ui/contexts/search"; +import { SEARCH_INDEX_PATH } from "@/lib/search.shared"; +import { createStaticSearchClient } from "@/lib/static-search-client"; import { buildDocsApiPath } from "@/lib/url-base"; const tags = [ @@ -27,11 +30,19 @@ const tags = [ export function CustomSearchDialog(props: SharedProps) { const [tag, setTag] = useState(); + const client = + process.env.NODE_ENV === "production" + ? createStaticSearchClient({ + from: SEARCH_INDEX_PATH, + tag, + }) + : fetchClient({ + api: buildDocsApiPath("search"), + tag, + }); const { search, setSearch, query } = useDocsSearch({ - type: "fetch", - api: buildDocsApiPath("search"), - delayMs: 500, - tag, + client, + delayMs: 100, }); return ( diff --git a/src/lib/search-index.ts b/src/lib/search-index.ts new file mode 100644 index 0000000..3c834a3 --- /dev/null +++ b/src/lib/search-index.ts @@ -0,0 +1,126 @@ +import Search from "flexsearch"; + +export type StaticSearchDocument = { + id: string; + breadcrumbs?: string[]; + content: string; + page_id: string; + tags: string[]; + type: "page" | "heading" | "text"; + url: string; +}; + +export type StaticSearchExport = { + type: "default"; + raw: Record; +}; + +export type StaticSearchResult = { + id: string; + breadcrumbs?: string[]; + content: string; + type: "page" | "heading" | "text"; + url: string; +}; + +function normalizeTag(tag?: string | string[]): string[] { + if (!tag) return []; + return Array.isArray(tag) ? tag : [tag]; +} + +function createSearchDocument() { + return new Search.Document({ + // Keep the exported index under Cloudflare's asset size limit. + tokenize: "strict", + resolution: 1, + document: { + id: "id", + index: ["content"], + tag: ["tags"], + store: ["id", "page_id", "type", "content", "breadcrumbs", "tags", "url"], + }, + }); +} + +export function buildStaticSearchIndex(documents: StaticSearchDocument[]) { + const index = createSearchDocument(); + for (const document of documents) { + index.add(document.id, document); + } + return index; +} + +export function exportStaticSearchIndex(index: ReturnType): StaticSearchExport { + const raw: Record = {}; + index.export((key, value) => { + raw[key] = value; + }); + return { + type: "default", + raw, + }; +} + +export function importStaticSearchIndex(data: StaticSearchExport) { + const index = createSearchDocument(); + for (const [key, value] of Object.entries(data.raw)) { + index.import(key, value); + } + return index; +} + +export async function searchStaticSearchIndex( + index: ReturnType, + query: string, + tag?: string | string[], + limit = 60, +): Promise { + const tags = normalizeTag(tag); + const groups = await index.searchAsync(query, { + index: "content", + limit, + tag: tags.length > 0 ? { tags } : undefined, + }); + + if (groups.length === 0) return []; + + const results = groups[0]?.result ?? []; + const grouped = new Map(); + + for (const id of results) { + const doc = index.get(id) as StaticSearchDocument | null; + if (!doc) continue; + + const items = grouped.get(doc.page_id) ?? []; + if (doc.type !== "page") { + items.push(doc); + } + grouped.set(doc.page_id, items); + } + + const output: StaticSearchResult[] = []; + for (const [pageId, items] of grouped) { + const page = index.get(pageId) as StaticSearchDocument | null; + if (!page) continue; + + output.push({ + id: pageId, + breadcrumbs: page.breadcrumbs, + content: page.content, + type: "page", + url: page.url, + }); + + for (const item of items) { + output.push({ + id: item.id, + breadcrumbs: item.breadcrumbs, + content: item.content, + type: item.type, + url: item.url, + }); + } + } + + return output; +} diff --git a/src/lib/search.shared.ts b/src/lib/search.shared.ts new file mode 100644 index 0000000..4aadfd1 --- /dev/null +++ b/src/lib/search.shared.ts @@ -0,0 +1,4 @@ +import { buildDocsPath } from "./url-base"; + +export const SEARCH_INDEX_FILENAME = "search-index.json"; +export const SEARCH_INDEX_PATH = buildDocsPath(SEARCH_INDEX_FILENAME); diff --git a/src/lib/search.ts b/src/lib/search.ts new file mode 100644 index 0000000..5f8c431 --- /dev/null +++ b/src/lib/search.ts @@ -0,0 +1,128 @@ +import { flexsearchFromSource } from "fumadocs-core/search/flexsearch"; +import type { InferPageType } from "fumadocs-core/source"; +import { source } from "./source"; +import type { StaticSearchDocument } from "./search-index"; + +const SDK_SEARCH_TAGS = new Set(["ios", "android", "flutter", "expo", "react-native"]); + +function getPrimaryPathSegment(page: InferPageType): string | undefined { + return page.url.replace(/^\/docs\/?/, "").split("/").filter(Boolean)[0]; +} + +function getSearchTags(page: InferPageType): string[] { + const segment = getPrimaryPathSegment(page); + return segment && SDK_SEARCH_TAGS.has(segment) ? [segment] : []; +} + +async function getStructuredData(page: InferPageType) { + if ("structuredData" in page.data) { + return typeof page.data.structuredData === "function" + ? await page.data.structuredData() + : page.data.structuredData; + } + + if ("load" in page.data && typeof page.data.load === "function") { + return (await page.data.load()).structuredData; + } + + throw new Error(`Cannot build search index for ${page.url}: missing structured data.`); +} + +type SearchPageIndex = { + id: string; + breadcrumbs?: string[]; + description?: string; + structuredData: Awaited>; + tag?: string | string[]; + title: string; + url: string; +}; + +function buildSearchDocuments(indexes: SearchPageIndex[]): StaticSearchDocument[] { + const documents: StaticSearchDocument[] = []; + + for (const page of indexes) { + const tags = Array.isArray(page.tag) ? page.tag : page.tag ? [page.tag] : []; + let nextId = 0; + const createId = () => `${page.id}-${nextId++}`; + + documents.push({ + id: page.id, + breadcrumbs: page.breadcrumbs, + content: page.title, + page_id: page.id, + tags, + type: "page", + url: page.url, + }); + + if (page.description) { + documents.push({ + id: createId(), + content: page.description, + page_id: page.id, + tags, + type: "text", + url: page.url, + }); + } + + for (const heading of page.structuredData.headings) { + documents.push({ + id: createId(), + content: heading.content, + page_id: page.id, + tags, + type: "heading", + url: `${page.url}#${heading.id}`, + }); + } + + for (const content of page.structuredData.contents) { + documents.push({ + id: createId(), + content: content.content, + page_id: page.id, + tags, + type: "text", + url: content.heading ? `${page.url}#${content.heading}` : page.url, + }); + } + } + + return documents; +} + +export function createDocsSearchApi() { + return flexsearchFromSource(source, { + async buildIndex(page) { + const fileName = page.path.split("/").at(-1) ?? page.path; + return { + id: page.url, + title: page.data.title ?? fileName.replace(/\.[^.]+$/, ""), + description: page.data.description, + structuredData: await getStructuredData(page), + tag: getSearchTags(page), + url: page.url, + }; + }, + }); +} + +export async function buildDocsSearchDocuments() { + const pages = await Promise.all( + source.getPages().map(async (page) => { + const fileName = page.path.split("/").at(-1) ?? page.path; + return { + id: page.url, + title: page.data.title ?? fileName.replace(/\.[^.]+$/, ""), + description: page.data.description, + structuredData: await getStructuredData(page), + tag: getSearchTags(page), + url: page.url, + } satisfies SearchPageIndex; + }), + ); + + return buildSearchDocuments(pages); +} diff --git a/src/lib/static-search-client.ts b/src/lib/static-search-client.ts new file mode 100644 index 0000000..9c5741f --- /dev/null +++ b/src/lib/static-search-client.ts @@ -0,0 +1,50 @@ +"use client"; + +import type { SearchClient } from "fumadocs-core/search/client"; +import { + importStaticSearchIndex, + searchStaticSearchIndex, + type StaticSearchExport, +} from "./search-index"; + +const indexCache = new Map>>(); + +type StaticSearchClientOptions = { + from: string; + tag?: string | string[]; +}; + +function loadIndex(from: string) { + let promise = indexCache.get(from); + if (!promise) { + promise = fetch(from) + .then(async (response) => { + if (!response.ok) { + throw new Error(`failed to fetch search index from ${from}`); + } + return response.json() as Promise; + }) + .then((payload) => importStaticSearchIndex(payload)) + .catch((error) => { + indexCache.delete(from); + throw error; + }); + indexCache.set(from, promise); + } + + return promise; +} + +export function createStaticSearchClient(options: StaticSearchClientOptions): SearchClient { + const { from, tag } = options; + + return { + deps: [from, tag], + async search(query) { + if (!query) return []; + + const index = await loadIndex(from); + return searchStaticSearchIndex(index, query, tag); + }, + }; +} diff --git a/src/routes/api/search.ts b/src/routes/api/search.ts index e6167c4..caf64a1 100644 --- a/src/routes/api/search.ts +++ b/src/routes/api/search.ts @@ -1,17 +1,16 @@ import { createFileRoute } from "@tanstack/react-router"; -import { source } from "@/lib/source"; -import { createFromSource } from "fumadocs-core/search/server"; +import { createDocsSearchApi } from "@/lib/search"; let server: - | ReturnType + | ReturnType | null = null; function getServer() { if (server) return server; - // Building the local Orama index is expensive; keep it off the main dev - // startup path and only initialize it when /api/search is actually hit. - server = createFromSource(source, { language: "english" }); + // In development, the fetch-based search client still hits this route. + // The production client loads a prebuilt FlexSearch index instead. + server = createDocsSearchApi(); return server; } From fa01d4a20018490ed506e9538b02112f6dc9c0c9 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Mon, 13 Apr 2026 16:52:31 -0700 Subject: [PATCH 02/11] Scope SDK search and add Superchat shortcut --- src/components/SearchDialog.tsx | 62 +++++++++++++++++++++++++-------- src/lib/search.shared.ts | 45 ++++++++++++++++++++++++ src/lib/search.ts | 10 ++---- src/lib/superchat.ts | 25 +++++++++++++ 4 files changed, 120 insertions(+), 22 deletions(-) create mode 100644 src/lib/superchat.ts diff --git a/src/components/SearchDialog.tsx b/src/components/SearchDialog.tsx index 94677ba..1ca2b33 100644 --- a/src/components/SearchDialog.tsx +++ b/src/components/SearchDialog.tsx @@ -1,6 +1,7 @@ "use client"; -import { useState } from "react"; +import { MessageSquarePlus } from "lucide-react"; +import { useEffect, useEffectEvent, useState } from "react"; import { SearchDialog, SearchDialogOverlay, @@ -17,19 +18,19 @@ import { import { useDocsSearch } from "fumadocs-core/search/client"; import { fetchClient } from "fumadocs-core/search/client/fetch"; import type { SharedProps } from "fumadocs-ui/contexts/search"; -import { SEARCH_INDEX_PATH } from "@/lib/search.shared"; +import { + getSearchGroupsForScope, + SEARCH_INDEX_PATH, + SEARCH_SCOPE_OPTIONS, + type SearchScope, +} from "@/lib/search.shared"; import { createStaticSearchClient } from "@/lib/static-search-client"; +import { openSuperchat } from "@/lib/superchat"; import { buildDocsApiPath } from "@/lib/url-base"; -const tags = [ - { name: "iOS", value: "ios" }, - { name: "Android", value: "android" }, - { name: "Flutter", value: "flutter" }, - { name: "Expo", value: "expo" }, -]; - export function CustomSearchDialog(props: SharedProps) { - const [tag, setTag] = useState(); + const [scope, setScope] = useState(); + const tag = getSearchGroupsForScope(scope); const client = process.env.NODE_ENV === "production" ? createStaticSearchClient({ @@ -44,6 +45,25 @@ export function CustomSearchDialog(props: SharedProps) { client, delayMs: 100, }); + const launchSuperchat = useEffectEvent((message = search) => { + const seededMessage = message.trim(); + if (!openSuperchat(seededMessage)) return; + props.onOpenChange(false); + }); + + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + if (event.isComposing) return; + if (!(event.metaKey || event.ctrlKey)) return; + if (event.key.toLowerCase() !== "i") return; + + event.preventDefault(); + launchSuperchat(); + }; + + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [launchSuperchat]); return ( @@ -52,14 +72,28 @@ export function CustomSearchDialog(props: SharedProps) { + - - {tags.map((t) => ( - - {t.name} + setScope(value as SearchScope | undefined)} + allowClear + > + {SEARCH_SCOPE_OPTIONS.map((option) => ( + + {option.name} ))} diff --git a/src/lib/search.shared.ts b/src/lib/search.shared.ts index 4aadfd1..5309135 100644 --- a/src/lib/search.shared.ts +++ b/src/lib/search.shared.ts @@ -2,3 +2,48 @@ import { buildDocsPath } from "./url-base"; export const SEARCH_INDEX_FILENAME = "search-index.json"; export const SEARCH_INDEX_PATH = buildDocsPath(SEARCH_INDEX_FILENAME); + +export const SHARED_SEARCH_GROUP = "shared"; +export const COMMUNITY_SEARCH_GROUP = "community"; +export const SDK_SEARCH_GROUPS = [ + "ios", + "android", + "flutter", + "expo", + "react-native", +] as const; + +export type SearchScope = (typeof SDK_SEARCH_GROUPS)[number]; +export type SearchGroup = + | typeof SHARED_SEARCH_GROUP + | typeof COMMUNITY_SEARCH_GROUP + | SearchScope; + +export const SEARCH_SCOPE_OPTIONS: Array<{ name: string; value: SearchScope }> = [ + { name: "iOS SDK", value: "ios" }, + { name: "Android SDK", value: "android" }, + { name: "Flutter SDK", value: "flutter" }, + { name: "Expo SDK", value: "expo" }, + { name: "React Native SDK", value: "react-native" }, +]; + +const SDK_SEARCH_GROUP_SET = new Set(SDK_SEARCH_GROUPS); + +export function getSearchGroupsForScope(scope?: SearchScope): SearchGroup[] | undefined { + if (!scope) return undefined; + return [SHARED_SEARCH_GROUP, scope]; +} + +export function getSearchGroupFromUrl(url: string): SearchGroup { + const segment = url.replace(/^\/docs\/?/, "").split("/").filter(Boolean)[0]; + + if (segment === COMMUNITY_SEARCH_GROUP) { + return COMMUNITY_SEARCH_GROUP; + } + + if (segment && SDK_SEARCH_GROUP_SET.has(segment)) { + return segment as SearchScope; + } + + return SHARED_SEARCH_GROUP; +} diff --git a/src/lib/search.ts b/src/lib/search.ts index 5f8c431..1424220 100644 --- a/src/lib/search.ts +++ b/src/lib/search.ts @@ -1,17 +1,11 @@ import { flexsearchFromSource } from "fumadocs-core/search/flexsearch"; import type { InferPageType } from "fumadocs-core/source"; +import { getSearchGroupFromUrl } from "./search.shared"; import { source } from "./source"; import type { StaticSearchDocument } from "./search-index"; -const SDK_SEARCH_TAGS = new Set(["ios", "android", "flutter", "expo", "react-native"]); - -function getPrimaryPathSegment(page: InferPageType): string | undefined { - return page.url.replace(/^\/docs\/?/, "").split("/").filter(Boolean)[0]; -} - function getSearchTags(page: InferPageType): string[] { - const segment = getPrimaryPathSegment(page); - return segment && SDK_SEARCH_TAGS.has(segment) ? [segment] : []; + return [getSearchGroupFromUrl(page.url)]; } async function getStructuredData(page: InferPageType) { diff --git a/src/lib/superchat.ts b/src/lib/superchat.ts new file mode 100644 index 0000000..c069d36 --- /dev/null +++ b/src/lib/superchat.ts @@ -0,0 +1,25 @@ +"use client"; + +const SUPERCHAT_ORIGIN = "https://superchat-production-416f.up.railway.app"; +const SUPERCHAT_FRAME_ID = "arona-frame"; + +function getSuperchatWindow(): Window | null { + const frame = document.getElementById(SUPERCHAT_FRAME_ID); + if (!(frame instanceof HTMLIFrameElement)) return null; + return frame.contentWindow; +} + +export function openSuperchat(message = ""): boolean { + const chatWindow = getSuperchatWindow(); + if (!chatWindow) return false; + + chatWindow.postMessage( + { + type: "ask", + message, + }, + SUPERCHAT_ORIGIN, + ); + + return true; +} From bb31011597f530a8de09a70cec0c23dc3bdb2905 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Mon, 13 Apr 2026 16:53:03 -0700 Subject: [PATCH 03/11] Shorten search scope labels --- src/lib/search.shared.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/search.shared.ts b/src/lib/search.shared.ts index 5309135..ea9c946 100644 --- a/src/lib/search.shared.ts +++ b/src/lib/search.shared.ts @@ -20,11 +20,11 @@ export type SearchGroup = | SearchScope; export const SEARCH_SCOPE_OPTIONS: Array<{ name: string; value: SearchScope }> = [ - { name: "iOS SDK", value: "ios" }, - { name: "Android SDK", value: "android" }, - { name: "Flutter SDK", value: "flutter" }, - { name: "Expo SDK", value: "expo" }, - { name: "React Native SDK", value: "react-native" }, + { name: "iOS", value: "ios" }, + { name: "Android", value: "android" }, + { name: "Flutter", value: "flutter" }, + { name: "Expo", value: "expo" }, + { name: "React Native", value: "react-native" }, ]; const SDK_SEARCH_GROUP_SET = new Set(SDK_SEARCH_GROUPS); From 436f8818060bb16e2b1856114f56d750331b9213 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Mon, 13 Apr 2026 16:58:16 -0700 Subject: [PATCH 04/11] Close Superchat from search hotkeys --- src/components/SearchDialog.tsx | 13 ++++++++++-- src/lib/superchat.ts | 35 ++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/components/SearchDialog.tsx b/src/components/SearchDialog.tsx index 1ca2b33..ae35f32 100644 --- a/src/components/SearchDialog.tsx +++ b/src/components/SearchDialog.tsx @@ -25,7 +25,7 @@ import { type SearchScope, } from "@/lib/search.shared"; import { createStaticSearchClient } from "@/lib/static-search-client"; -import { openSuperchat } from "@/lib/superchat"; +import { closeSuperchat, isSuperchatOpen, openSuperchat } from "@/lib/superchat"; import { buildDocsApiPath } from "@/lib/url-base"; export function CustomSearchDialog(props: SharedProps) { @@ -55,7 +55,16 @@ export function CustomSearchDialog(props: SharedProps) { const onKeyDown = (event: KeyboardEvent) => { if (event.isComposing) return; if (!(event.metaKey || event.ctrlKey)) return; - if (event.key.toLowerCase() !== "i") return; + const key = event.key.toLowerCase(); + if (key !== "i" && key !== "k") return; + + if (isSuperchatOpen()) { + event.preventDefault(); + closeSuperchat(); + return; + } + + if (key !== "i") return; event.preventDefault(); launchSuperchat(); diff --git a/src/lib/superchat.ts b/src/lib/superchat.ts index c069d36..9c0dfcf 100644 --- a/src/lib/superchat.ts +++ b/src/lib/superchat.ts @@ -2,11 +2,40 @@ const SUPERCHAT_ORIGIN = "https://superchat-production-416f.up.railway.app"; const SUPERCHAT_FRAME_ID = "arona-frame"; +const SUPERCHAT_CLOSED_WIDTH = "74px"; +const SUPERCHAT_CLOSED_HEIGHT = "74px"; -function getSuperchatWindow(): Window | null { +function getSuperchatFrame(): HTMLIFrameElement | null { const frame = document.getElementById(SUPERCHAT_FRAME_ID); - if (!(frame instanceof HTMLIFrameElement)) return null; - return frame.contentWindow; + return frame instanceof HTMLIFrameElement ? frame : null; +} + +function getSuperchatWindow(): Window | null { + return getSuperchatFrame()?.contentWindow ?? null; +} + +export function isSuperchatOpen(): boolean { + const frame = getSuperchatFrame(); + if (!frame) return false; + + return ( + frame.style.width !== SUPERCHAT_CLOSED_WIDTH || + frame.style.height !== SUPERCHAT_CLOSED_HEIGHT + ); +} + +export function closeSuperchat(): boolean { + const chatWindow = getSuperchatWindow(); + if (!chatWindow) return false; + + chatWindow.postMessage( + { + type: "close", + }, + SUPERCHAT_ORIGIN, + ); + + return true; } export function openSuperchat(message = ""): boolean { From cc2bc61aef62887b85578e9b123a5f85862755d3 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Mon, 13 Apr 2026 17:21:25 -0700 Subject: [PATCH 05/11] Polish docs AI entry and add Agentation --- bun.lock | 3 +++ package.json | 1 + src/components/SearchDialog.tsx | 18 ++++++++++++------ src/routes/__root.tsx | 2 ++ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bun.lock b/bun.lock index 0b92017..8462609 100644 --- a/bun.lock +++ b/bun.lock @@ -12,6 +12,7 @@ "@tanstack/react-router": "1.163.3", "@tanstack/react-router-devtools": "1.163.3", "@tanstack/react-start": "1.166.1", + "agentation": "^3.0.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "flexsearch": "^0.8.212", @@ -691,6 +692,8 @@ "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + "agentation": ["agentation@3.0.2", "", { "peerDependencies": { "react": ">=18.0.0", "react-dom": ">=18.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-iGzBxFVTuZEIKzLY6AExSLAQH6i6SwxV4pAu7v7m3X6bInZ7qlZXAwrEqyc4+EfP4gM7z2RXBF6SF4DeH0f2lA=="], + "ansi-colors": ["ansi-colors@4.1.3", "", {}, "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw=="], "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], diff --git a/package.json b/package.json index 7829fbb..9d96851 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "@tanstack/react-router": "1.163.3", "@tanstack/react-router-devtools": "1.163.3", "@tanstack/react-start": "1.166.1", + "agentation": "^3.0.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "flexsearch": "^0.8.212", diff --git a/src/components/SearchDialog.tsx b/src/components/SearchDialog.tsx index ae35f32..d7e1cf5 100644 --- a/src/components/SearchDialog.tsx +++ b/src/components/SearchDialog.tsx @@ -1,6 +1,6 @@ "use client"; -import { MessageSquarePlus } from "lucide-react"; +import { MessageCircle } from "lucide-react"; import { useEffect, useEffectEvent, useState } from "react"; import { SearchDialog, @@ -9,7 +9,6 @@ import { SearchDialogHeader, SearchDialogIcon, SearchDialogInput, - SearchDialogClose, SearchDialogList, SearchDialogFooter, TagsList, @@ -84,14 +83,21 @@ export function CustomSearchDialog(props: SharedProps) { - diff --git a/src/routes/__root.tsx b/src/routes/__root.tsx index 6428ed8..4c36d02 100644 --- a/src/routes/__root.tsx +++ b/src/routes/__root.tsx @@ -10,6 +10,7 @@ import { } from "@tanstack/react-router"; import * as React from "react"; import { useRef, useMemo } from "react"; +import { Agentation } from "agentation"; import appCss from "@/styles/app.css?url"; import { RootProvider as BaseRootProvider } from "fumadocs-ui/provider/base"; import { FrameworkProvider } from "fumadocs-core/framework"; @@ -207,6 +208,7 @@ function RootDocument({ children }: { children: React.ReactNode }) { {children} + {import.meta.env.DEV ? : null} From 97dfa666bca308de92edf2101bd275813cb2c5f7 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Mon, 13 Apr 2026 17:40:22 -0700 Subject: [PATCH 06/11] Fix static search parity and index serialization --- scripts/generate-search-index.ts | 2 +- src/lib/search-index.test.ts | 61 ++++++++++++++++++++++++++++++++ src/lib/search-index.ts | 30 ++++++++++------ src/lib/search.ts | 29 +++++++++++++++ src/lib/static-search-client.ts | 3 +- 5 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 src/lib/search-index.test.ts diff --git a/scripts/generate-search-index.ts b/scripts/generate-search-index.ts index caa9c69..dec5830 100644 --- a/scripts/generate-search-index.ts +++ b/scripts/generate-search-index.ts @@ -39,7 +39,7 @@ async function main() { const outputPath = path.join(outputDir, SEARCH_INDEX_FILENAME); const documents = await buildDocsSearchDocuments(); const index = buildStaticSearchIndex(documents); - const payload = JSON.stringify(exportStaticSearchIndex(index)); + const payload = JSON.stringify(await exportStaticSearchIndex(index)); await fs.mkdir(outputDir, { recursive: true }); await fs.writeFile(outputPath, payload); diff --git a/src/lib/search-index.test.ts b/src/lib/search-index.test.ts new file mode 100644 index 0000000..09af354 --- /dev/null +++ b/src/lib/search-index.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { + buildStaticSearchIndex, + exportStaticSearchIndex, + importStaticSearchIndex, + searchStaticSearchIndex, + type StaticSearchDocument, +} from "./search-index"; + +const documents: StaticSearchDocument[] = [ + { + id: "/docs/ios/purchase-controller", + page_id: "/docs/ios/purchase-controller", + type: "page", + content: "Purchase Controller", + breadcrumbs: ["Docs", "iOS"], + tags: ["ios"], + url: "/docs/ios/purchase-controller", + }, + { + id: "/docs/ios/purchase-controller-0", + page_id: "/docs/ios/purchase-controller", + type: "heading", + content: "PurchaseController", + tags: ["ios"], + url: "/docs/ios/purchase-controller#purchase-controller", + }, + { + id: "/docs/ios/purchase-controller-1", + page_id: "/docs/ios/purchase-controller", + type: "text", + content: "Use PurchaseController to manage purchases.", + tags: ["ios"], + url: "/docs/ios/purchase-controller#purchase-controller", + }, +]; + +describe("static search index", () => { + test("round-trips exported indexes", async () => { + const index = buildStaticSearchIndex(documents); + const exported = await exportStaticSearchIndex(index); + const restored = await importStaticSearchIndex(exported); + const results = await searchStaticSearchIndex(restored, "purchase controller", "ios"); + + expect(exported.raw).not.toEqual({}); + expect(results.length).toBeGreaterThan(0); + expect(results[0]).toMatchObject({ + id: "/docs/ios/purchase-controller", + breadcrumbs: ["Docs", "iOS"], + url: "/docs/ios/purchase-controller", + }); + }); + + test("keeps prefix search and highlights matched content", async () => { + const index = buildStaticSearchIndex(documents); + const results = await searchStaticSearchIndex(index, "purch", "ios"); + + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.content).toContain(""); + }); +}); diff --git a/src/lib/search-index.ts b/src/lib/search-index.ts index 3c834a3..70c3f9f 100644 --- a/src/lib/search-index.ts +++ b/src/lib/search-index.ts @@ -1,4 +1,5 @@ import Search from "flexsearch"; +import { createContentHighlighter } from "fumadocs-core/search"; export type StaticSearchDocument = { id: string; @@ -23,6 +24,8 @@ export type StaticSearchResult = { url: string; }; +export type StaticSearchIndex = ReturnType; + function normalizeTag(tag?: string | string[]): string[] { if (!tag) return []; return Array.isArray(tag) ? tag : [tag]; @@ -30,8 +33,8 @@ function normalizeTag(tag?: string | string[]): string[] { function createSearchDocument() { return new Search.Document({ - // Keep the exported index under Cloudflare's asset size limit. - tokenize: "strict", + // Preserve prefix search while still keeping the exported index small enough for static delivery. + tokenize: "full", resolution: 1, document: { id: "id", @@ -42,7 +45,7 @@ function createSearchDocument() { }); } -export function buildStaticSearchIndex(documents: StaticSearchDocument[]) { +export function buildStaticSearchIndex(documents: StaticSearchDocument[]): StaticSearchIndex { const index = createSearchDocument(); for (const document of documents) { index.add(document.id, document); @@ -50,27 +53,33 @@ export function buildStaticSearchIndex(documents: StaticSearchDocument[]) { return index; } -export function exportStaticSearchIndex(index: ReturnType): StaticSearchExport { +export async function exportStaticSearchIndex(index: StaticSearchIndex): Promise { const raw: Record = {}; - index.export((key, value) => { + const maybePromise = index.export((key, value) => { raw[key] = value; }); + await maybePromise; return { type: "default", raw, }; } -export function importStaticSearchIndex(data: StaticSearchExport) { +export async function importStaticSearchIndex(data: StaticSearchExport): Promise { const index = createSearchDocument(); + const imports: Promise[] = []; for (const [key, value] of Object.entries(data.raw)) { - index.import(key, value); + const maybePromise = index.import(key, value); + if (maybePromise?.then) { + imports.push(maybePromise); + } } + await Promise.all(imports); return index; } export async function searchStaticSearchIndex( - index: ReturnType, + index: StaticSearchIndex, query: string, tag?: string | string[], limit = 60, @@ -85,6 +94,7 @@ export async function searchStaticSearchIndex( if (groups.length === 0) return []; const results = groups[0]?.result ?? []; + const highlighter = createContentHighlighter(query); const grouped = new Map(); for (const id of results) { @@ -106,7 +116,7 @@ export async function searchStaticSearchIndex( output.push({ id: pageId, breadcrumbs: page.breadcrumbs, - content: page.content, + content: highlighter.highlightMarkdown(page.content), type: "page", url: page.url, }); @@ -115,7 +125,7 @@ export async function searchStaticSearchIndex( output.push({ id: item.id, breadcrumbs: item.breadcrumbs, - content: item.content, + content: highlighter.highlightMarkdown(item.content), type: item.type, url: item.url, }); diff --git a/src/lib/search.ts b/src/lib/search.ts index 1424220..e338b20 100644 --- a/src/lib/search.ts +++ b/src/lib/search.ts @@ -1,4 +1,5 @@ import { flexsearchFromSource } from "fumadocs-core/search/flexsearch"; +import { findPath } from "fumadocs-core/page-tree"; import type { InferPageType } from "fumadocs-core/source"; import { getSearchGroupFromUrl } from "./search.shared"; import { source } from "./source"; @@ -22,6 +23,32 @@ async function getStructuredData(page: InferPageType) { throw new Error(`Cannot build search index for ${page.url}: missing structured data.`); } +function isBreadcrumbItem(item: unknown): item is string { + return typeof item === "string" && item.length > 0; +} + +function getBreadcrumbs(page: InferPageType) { + const pageTree = source.getPageTree(page.locale); + const path = findPath(pageTree.children, (node) => node.type === "page" && node.url === page.url); + + if (!path) return undefined; + + const breadcrumbs: string[] = []; + path.pop(); + + if (isBreadcrumbItem(pageTree.name)) { + breadcrumbs.push(pageTree.name); + } + + for (const segment of path) { + if (isBreadcrumbItem(segment.name)) { + breadcrumbs.push(segment.name); + } + } + + return breadcrumbs; +} + type SearchPageIndex = { id: string; breadcrumbs?: string[]; @@ -96,6 +123,7 @@ export function createDocsSearchApi() { title: page.data.title ?? fileName.replace(/\.[^.]+$/, ""), description: page.data.description, structuredData: await getStructuredData(page), + breadcrumbs: getBreadcrumbs(page), tag: getSearchTags(page), url: page.url, }; @@ -112,6 +140,7 @@ export async function buildDocsSearchDocuments() { title: page.data.title ?? fileName.replace(/\.[^.]+$/, ""), description: page.data.description, structuredData: await getStructuredData(page), + breadcrumbs: getBreadcrumbs(page), tag: getSearchTags(page), url: page.url, } satisfies SearchPageIndex; diff --git a/src/lib/static-search-client.ts b/src/lib/static-search-client.ts index 9c5741f..3fb06b5 100644 --- a/src/lib/static-search-client.ts +++ b/src/lib/static-search-client.ts @@ -4,10 +4,11 @@ import type { SearchClient } from "fumadocs-core/search/client"; import { importStaticSearchIndex, searchStaticSearchIndex, + type StaticSearchIndex, type StaticSearchExport, } from "./search-index"; -const indexCache = new Map>>(); +const indexCache = new Map>(); type StaticSearchClientOptions = { from: string; From c89d158d517532d278c139fb8a78e5829340ae9e Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Tue, 14 Apr 2026 13:35:07 -0700 Subject: [PATCH 07/11] Simplify docs search to a strict static monolith --- scripts/generate-search-index.ts | 4 +- src/components/SearchDialog.tsx | 38 ++++++- src/lib/search-index.test.ts | 48 +++++++- src/lib/search-index.ts | 190 +++++++++++++++++++++++++++---- src/lib/search.shared.ts | 38 ++++++- src/lib/static-search-client.ts | 49 ++++++-- 6 files changed, 320 insertions(+), 47 deletions(-) diff --git a/scripts/generate-search-index.ts b/scripts/generate-search-index.ts index dec5830..4a5c214 100644 --- a/scripts/generate-search-index.ts +++ b/scripts/generate-search-index.ts @@ -38,10 +38,12 @@ async function main() { const outputDir = path.join(DIST_CLIENT, "docs"); const outputPath = path.join(outputDir, SEARCH_INDEX_FILENAME); const documents = await buildDocsSearchDocuments(); + + await fs.rm(path.join(outputDir, "assets", "search"), { force: true, recursive: true }); + await fs.mkdir(outputDir, { recursive: true }); const index = buildStaticSearchIndex(documents); const payload = JSON.stringify(await exportStaticSearchIndex(index)); - await fs.mkdir(outputDir, { recursive: true }); await fs.writeFile(outputPath, payload); const bytes = Buffer.byteLength(payload); diff --git a/src/components/SearchDialog.tsx b/src/components/SearchDialog.tsx index d7e1cf5..203dab2 100644 --- a/src/components/SearchDialog.tsx +++ b/src/components/SearchDialog.tsx @@ -23,22 +23,34 @@ import { SEARCH_SCOPE_OPTIONS, type SearchScope, } from "@/lib/search.shared"; -import { createStaticSearchClient } from "@/lib/static-search-client"; +import { createStaticSearchClient, preloadStaticSearch } from "@/lib/static-search-client"; import { closeSuperchat, isSuperchatOpen, openSuperchat } from "@/lib/superchat"; import { buildDocsApiPath } from "@/lib/url-base"; +function scheduleIdle(callback: () => void) { + if (typeof window === "undefined") return () => undefined; + + if ("requestIdleCallback" in window) { + const handle = window.requestIdleCallback(() => callback()); + return () => window.cancelIdleCallback(handle); + } + + const handle = window.setTimeout(callback, 1_200); + return () => window.clearTimeout(handle); +} + export function CustomSearchDialog(props: SharedProps) { const [scope, setScope] = useState(); - const tag = getSearchGroupsForScope(scope); + const tags = getSearchGroupsForScope(scope); const client = process.env.NODE_ENV === "production" ? createStaticSearchClient({ from: SEARCH_INDEX_PATH, - tag, + tags, }) : fetchClient({ api: buildDocsApiPath("search"), - tag, + tag: tags, }); const { search, setSearch, query } = useDocsSearch({ client, @@ -50,6 +62,24 @@ export function CustomSearchDialog(props: SharedProps) { props.onOpenChange(false); }); + useEffect(() => { + if (process.env.NODE_ENV !== "production") return; + const saveData = Boolean( + ( + navigator as Navigator & { + connection?: { + saveData?: boolean; + }; + } + ).connection?.saveData, + ); + if (saveData) return; + + return scheduleIdle(() => { + void preloadStaticSearch(SEARCH_INDEX_PATH); + }); + }, []); + useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { if (event.isComposing) return; diff --git a/src/lib/search-index.test.ts b/src/lib/search-index.test.ts index 09af354..f19c0ea 100644 --- a/src/lib/search-index.test.ts +++ b/src/lib/search-index.test.ts @@ -33,6 +33,40 @@ const documents: StaticSearchDocument[] = [ tags: ["ios"], url: "/docs/ios/purchase-controller#purchase-controller", }, + { + id: "/docs/android/sdk-reference/PurchaseController", + page_id: "/docs/android/sdk-reference/PurchaseController", + type: "page", + content: "PurchaseController", + breadcrumbs: ["Docs", "Android", "SDK Reference"], + tags: ["android"], + url: "/docs/android/sdk-reference/PurchaseController", + }, + { + id: "/docs/android/sdk-reference/PurchaseController-0", + page_id: "/docs/android/sdk-reference/PurchaseController", + type: "text", + content: "PurchaseController is the main Android purchase API.", + tags: ["android"], + url: "/docs/android/sdk-reference/PurchaseController#overview", + }, + { + id: "/docs/android/changelog", + page_id: "/docs/android/changelog", + type: "page", + content: "Changelog", + breadcrumbs: ["Docs", "Android"], + tags: ["android"], + url: "/docs/android/changelog", + }, + { + id: "/docs/android/changelog-0", + page_id: "/docs/android/changelog", + type: "text", + content: "Added PurchaseController improvements in version 2.0.", + tags: ["android"], + url: "/docs/android/changelog#v2", + }, ]; describe("static search index", () => { @@ -51,11 +85,21 @@ describe("static search index", () => { }); }); - test("keeps prefix search and highlights matched content", async () => { + test("highlights matched content for exact queries", async () => { const index = buildStaticSearchIndex(documents); - const results = await searchStaticSearchIndex(index, "purch", "ios"); + const results = await searchStaticSearchIndex(index, "purchase controller", "ios"); expect(results.length).toBeGreaterThan(0); expect(results[0]?.content).toContain(""); }); + + test("boosts exact SDK reference pages above changelog mentions", async () => { + const index = buildStaticSearchIndex(documents); + const results = await searchStaticSearchIndex(index, "purchase controller", "android"); + + expect(results[0]).toMatchObject({ + id: "/docs/android/sdk-reference/PurchaseController", + url: "/docs/android/sdk-reference/PurchaseController", + }); + }); }); diff --git a/src/lib/search-index.ts b/src/lib/search-index.ts index 70c3f9f..ba7b430 100644 --- a/src/lib/search-index.ts +++ b/src/lib/search-index.ts @@ -6,6 +6,7 @@ export type StaticSearchDocument = { breadcrumbs?: string[]; content: string; page_id: string; + search_content?: string; tags: string[]; type: "page" | "heading" | "text"; url: string; @@ -20,6 +21,7 @@ export type StaticSearchResult = { id: string; breadcrumbs?: string[]; content: string; + score?: number; type: "page" | "heading" | "text"; url: string; }; @@ -31,16 +33,30 @@ function normalizeTag(tag?: string | string[]): string[] { return Array.isArray(tag) ? tag : [tag]; } +function expandSearchContent(value: string): string { + const normalized = value + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/[_/.-]+/g, " ") + .trim() + .replace(/\s+/g, " "); + + if (!normalized || normalized === value) { + return value; + } + + return `${value}\n${normalized}`; +} + function createSearchDocument() { return new Search.Document({ - // Preserve prefix search while still keeping the exported index small enough for static delivery. - tokenize: "full", + // "strict" keeps the static monolith small enough to ship as a single Cloudflare asset. + tokenize: "strict", resolution: 1, document: { id: "id", - index: ["content"], + index: ["search_content"], tag: ["tags"], - store: ["id", "page_id", "type", "content", "breadcrumbs", "tags", "url"], + store: ["id", "page_id", "type", "content", "breadcrumbs", "url"], }, }); } @@ -48,11 +64,116 @@ function createSearchDocument() { export function buildStaticSearchIndex(documents: StaticSearchDocument[]): StaticSearchIndex { const index = createSearchDocument(); for (const document of documents) { - index.add(document.id, document); + index.add(document.id, { + ...document, + search_content: document.search_content ?? expandSearchContent(document.content), + }); } return index; } +function normalizeForMatch(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim() + .replace(/\s+/g, " "); +} + +function normalizeCompact(value: string): string { + return normalizeForMatch(value).replace(/\s+/g, ""); +} + +function tokenizeForMatch(value: string): string[] { + const normalized = normalizeForMatch(value); + return normalized ? normalized.split(" ") : []; +} + +function scorePrefixCoverage(queryTerms: string[], candidateTerms: string[]): number { + if (queryTerms.length === 0 || candidateTerms.length === 0) return 0; + if (!queryTerms.every((term) => candidateTerms.some((candidate) => candidate.startsWith(term)))) { + return 0; + } + + return 500; +} + +function scoreUrlMatch(query: string, url: string): number { + const path = url.split("#")[0] ?? url; + const lastSegment = decodeURIComponent(path.split("/").filter(Boolean).at(-1) ?? ""); + const normalizedSegment = normalizeForMatch(lastSegment); + const compactSegment = normalizeCompact(lastSegment); + const normalizedQuery = normalizeForMatch(query); + const compactQuery = normalizeCompact(query); + + if (!normalizedSegment) return 0; + if (compactSegment === compactQuery) return 900; + if (normalizedSegment === normalizedQuery) return 850; + if (compactSegment.includes(compactQuery)) return 500; + if (normalizedSegment.includes(normalizedQuery)) return 450; + return 0; +} + +type GroupedPage = { + items: StaticSearchDocument[]; + page: StaticSearchDocument; + score: number; +}; + +function scoreGroupedPage(query: string, page: StaticSearchDocument, items: StaticSearchDocument[], firstRank: number) { + const normalizedQuery = normalizeForMatch(query); + const compactQuery = normalizeCompact(query); + const queryTerms = tokenizeForMatch(query); + const normalizedTitle = normalizeForMatch(page.content); + const compactTitle = normalizeCompact(page.content); + const titleTerms = tokenizeForMatch(page.content); + + let score = Math.max(0, 800 - firstRank * 6); + + if (compactTitle === compactQuery) { + score += 4_000; + } else if (normalizedTitle === normalizedQuery) { + score += 3_500; + } else { + if (compactTitle.includes(compactQuery)) score += 1_500; + if (normalizedTitle.includes(normalizedQuery)) score += 1_200; + } + + score += scorePrefixCoverage(queryTerms, titleTerms); + score += scoreUrlMatch(query, page.url); + + const headingMatches = items.filter((item) => item.type === "heading"); + for (const heading of headingMatches) { + const normalizedHeading = normalizeForMatch(heading.content); + const compactHeading = normalizeCompact(heading.content); + const headingTerms = tokenizeForMatch(heading.content); + + if (compactHeading === compactQuery) { + score += 2_200; + continue; + } + + if (normalizedHeading === normalizedQuery) { + score += 1_900; + continue; + } + + if (compactHeading.includes(compactQuery)) score += 750; + if (normalizedHeading.includes(normalizedQuery)) score += 650; + score += scorePrefixCoverage(queryTerms, headingTerms); + } + + if (!normalizedQuery.includes("changelog") && page.url.endsWith("/changelog")) { + score -= 400; + } + + if (page.url.includes("/sdk-reference/") && scoreUrlMatch(query, page.url) > 0) { + score += 300; + } + + return score; +} + export async function exportStaticSearchIndex(index: StaticSearchIndex): Promise { const raw: Record = {}; const maybePromise = index.export((key, value) => { @@ -85,9 +206,10 @@ export async function searchStaticSearchIndex( limit = 60, ): Promise { const tags = normalizeTag(tag); + const rawLimit = Math.max(limit * 4, 120); const groups = await index.searchAsync(query, { - index: "content", - limit, + index: "search_content", + limit: rawLimit, tag: tags.length > 0 ? { tags } : undefined, }); @@ -95,42 +217,66 @@ export async function searchStaticSearchIndex( const results = groups[0]?.result ?? []; const highlighter = createContentHighlighter(query); - const grouped = new Map(); + const grouped = new Map(); - for (const id of results) { + for (const [rank, id] of results.entries()) { const doc = index.get(id) as StaticSearchDocument | null; if (!doc) continue; - const items = grouped.get(doc.page_id) ?? []; - if (doc.type !== "page") { - items.push(doc); + const entry = grouped.get(doc.page_id) ?? { + firstRank: rank, + items: [], + }; + + entry.firstRank = Math.min(entry.firstRank, rank); + if (doc.type === "page") { + entry.page = doc; + } else { + entry.items.push(doc); } - grouped.set(doc.page_id, items); + grouped.set(doc.page_id, entry); } - const output: StaticSearchResult[] = []; - for (const [pageId, items] of grouped) { - const page = index.get(pageId) as StaticSearchDocument | null; + const rankedPages: GroupedPage[] = []; + for (const [pageId, entry] of grouped) { + const page = entry.page ?? (index.get(pageId) as StaticSearchDocument | null); if (!page) continue; + rankedPages.push({ + items: entry.items, + page, + score: scoreGroupedPage(query, page, entry.items, entry.firstRank), + }); + } + + rankedPages.sort((left, right) => right.score - left.score); + + const output: StaticSearchResult[] = []; + for (const group of rankedPages) { output.push({ - id: pageId, - breadcrumbs: page.breadcrumbs, - content: highlighter.highlightMarkdown(page.content), + id: group.page.id, + breadcrumbs: group.page.breadcrumbs, + content: highlighter.highlightMarkdown(group.page.content), + score: group.score, type: "page", - url: page.url, + url: group.page.url, }); - for (const item of items) { + for (const [indexWithinPage, item] of group.items.entries()) { output.push({ id: item.id, breadcrumbs: item.breadcrumbs, content: highlighter.highlightMarkdown(item.content), + score: group.score - (indexWithinPage + 1) * 0.001, type: item.type, url: item.url, }); } + + if (output.length >= limit) { + break; + } } - return output; + return output.slice(0, limit); } diff --git a/src/lib/search.shared.ts b/src/lib/search.shared.ts index ea9c946..f279ea0 100644 --- a/src/lib/search.shared.ts +++ b/src/lib/search.shared.ts @@ -3,7 +3,11 @@ import { buildDocsPath } from "./url-base"; export const SEARCH_INDEX_FILENAME = "search-index.json"; export const SEARCH_INDEX_PATH = buildDocsPath(SEARCH_INDEX_FILENAME); -export const SHARED_SEARCH_GROUP = "shared"; +export const DASHBOARD_SEARCH_GROUP = "dashboard"; +export const WEB_CHECKOUT_SEARCH_GROUP = "web-checkout"; +export const INTEGRATIONS_SEARCH_GROUP = "integrations"; +export const SUPPORT_SEARCH_GROUP = "support"; +export const GENERAL_SEARCH_GROUP = "general"; export const COMMUNITY_SEARCH_GROUP = "community"; export const SDK_SEARCH_GROUPS = [ "ios", @@ -14,8 +18,14 @@ export const SDK_SEARCH_GROUPS = [ ] as const; export type SearchScope = (typeof SDK_SEARCH_GROUPS)[number]; +export type SearchCommonGroup = + | typeof DASHBOARD_SEARCH_GROUP + | typeof WEB_CHECKOUT_SEARCH_GROUP + | typeof INTEGRATIONS_SEARCH_GROUP + | typeof SUPPORT_SEARCH_GROUP + | typeof GENERAL_SEARCH_GROUP; export type SearchGroup = - | typeof SHARED_SEARCH_GROUP + | SearchCommonGroup | typeof COMMUNITY_SEARCH_GROUP | SearchScope; @@ -27,11 +37,23 @@ export const SEARCH_SCOPE_OPTIONS: Array<{ name: string; value: SearchScope }> = { name: "React Native", value: "react-native" }, ]; +export const COMMON_SEARCH_GROUPS: SearchCommonGroup[] = [ + DASHBOARD_SEARCH_GROUP, + WEB_CHECKOUT_SEARCH_GROUP, + INTEGRATIONS_SEARCH_GROUP, + SUPPORT_SEARCH_GROUP, + GENERAL_SEARCH_GROUP, +]; + const SDK_SEARCH_GROUP_SET = new Set(SDK_SEARCH_GROUPS); +const COMMON_SEARCH_GROUP_SET = new Set(COMMON_SEARCH_GROUPS); + +export function getSearchGroupsForScope(scope?: SearchScope): SearchGroup[] { + if (scope) { + return [scope, ...COMMON_SEARCH_GROUPS]; + } -export function getSearchGroupsForScope(scope?: SearchScope): SearchGroup[] | undefined { - if (!scope) return undefined; - return [SHARED_SEARCH_GROUP, scope]; + return [...COMMON_SEARCH_GROUPS, COMMUNITY_SEARCH_GROUP]; } export function getSearchGroupFromUrl(url: string): SearchGroup { @@ -45,5 +67,9 @@ export function getSearchGroupFromUrl(url: string): SearchGroup { return segment as SearchScope; } - return SHARED_SEARCH_GROUP; + if (segment && COMMON_SEARCH_GROUP_SET.has(segment)) { + return segment as SearchCommonGroup; + } + + return GENERAL_SEARCH_GROUP; } diff --git a/src/lib/static-search-client.ts b/src/lib/static-search-client.ts index 3fb06b5..a5a7395 100644 --- a/src/lib/static-search-client.ts +++ b/src/lib/static-search-client.ts @@ -7,24 +7,45 @@ import { type StaticSearchIndex, type StaticSearchExport, } from "./search-index"; +import type { SearchGroup } from "./search.shared"; const indexCache = new Map>(); +const INDEX_CACHE_NAME = "docs-static-search-index-v1"; type StaticSearchClientOptions = { from: string; - tag?: string | string[]; + tags?: SearchGroup[]; }; -function loadIndex(from: string) { +async function fetchIndexPayload(from: string): Promise { + if (typeof window !== "undefined" && "caches" in window) { + const cache = await window.caches.open(INDEX_CACHE_NAME); + const cachedResponse = await cache.match(from); + if (cachedResponse) { + return cachedResponse.json() as Promise; + } + + const response = await fetch(from, { cache: "force-cache" }); + if (!response.ok) { + throw new Error(`failed to fetch search index from ${from}`); + } + + void cache.put(from, response.clone()); + return response.json() as Promise; + } + + const response = await fetch(from, { cache: "force-cache" }); + if (!response.ok) { + throw new Error(`failed to fetch search index from ${from}`); + } + + return response.json() as Promise; +} + +async function loadIndex(from: string) { let promise = indexCache.get(from); if (!promise) { - promise = fetch(from) - .then(async (response) => { - if (!response.ok) { - throw new Error(`failed to fetch search index from ${from}`); - } - return response.json() as Promise; - }) + promise = fetchIndexPayload(from) .then((payload) => importStaticSearchIndex(payload)) .catch((error) => { indexCache.delete(from); @@ -36,16 +57,20 @@ function loadIndex(from: string) { return promise; } +export async function preloadStaticSearch(from: string): Promise { + await loadIndex(from); +} + export function createStaticSearchClient(options: StaticSearchClientOptions): SearchClient { - const { from, tag } = options; + const { from, tags } = options; return { - deps: [from, tag], + deps: [from, tags?.join(",")], async search(query) { if (!query) return []; const index = await loadIndex(from); - return searchStaticSearchIndex(index, query, tag); + return searchStaticSearchIndex(index, query, tags); }, }; } From c18f2afdf2f53c89e5443a7e78eebad3e16fb3ce Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Wed, 15 Apr 2026 16:13:00 -0700 Subject: [PATCH 08/11] Fix static search filtering and cache freshness --- package.json | 1 - scripts/search-benchmark.ts | 657 -------------------------------- src/lib/search.shared.ts | 25 +- src/lib/static-search-client.ts | 19 +- 4 files changed, 12 insertions(+), 690 deletions(-) delete mode 100644 scripts/search-benchmark.ts diff --git a/package.json b/package.json index 9d96851..4f7f2c4 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "sync:mixedbread": "mxbai vs sync $MIXEDBREAD_STORE_ID './content/docs' --ci", "preview": "vite preview", "preview:cf": "bun run build:cf && vite preview", - "benchmark:search": "bun run scripts/search-benchmark.ts", "deploy": "bun run build:cf && wrangler deploy", "deploy:staging": "bun run build:cf:staging && wrangler deploy", "cf:typegen": "wrangler types", diff --git a/scripts/search-benchmark.ts b/scripts/search-benchmark.ts deleted file mode 100644 index f1faffa..0000000 --- a/scripts/search-benchmark.ts +++ /dev/null @@ -1,657 +0,0 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import path from "node:path"; - -type QueryCase = { - name: string; - query: string; - expectedText?: string; -}; - -type SearchRequestRecord = { - requestId: string; - url: string; - query: string | null; - kind: "api" | "index"; - method: string; - status?: number; - observedAtMs: number; -}; - -type RawQueryResult = { - name: string; - query: string; - expectedText?: string; - inputSetMs: number; - firstTransportRequestSeenMs: number | null; - firstVisibleResultMs: number | null; - transportRequestCount: number; - transportRequests: Array<{ - kind: "api" | "index"; - query: string | null; - url: string; - status?: number; - seenOffsetMs: number | null; - }>; -}; - -type TypingSimulationResult = { - query: string; - cadenceMs: number; - totalWallMs: number; - firstTransportRequestSeenMs: number | null; - finalTransportRequestSeenMs: number | null; - finalResultVisibleMs: number | null; - transportRequests: Array<{ - kind: "api" | "index"; - query: string | null; - url: string; - status?: number; - seenOffsetMs: number | null; - }>; -}; - -type BenchmarkResult = { - url: string; - generatedAt: string; - outputPath: string; - sessionName: string; - userAgent: string; - viewport: { - width: number; - height: number; - deviceScaleFactor: number; - }; - navigation: { - openMs: number; - searchButtonReadyMs: number; - metrics: Record | null; - }; - searchDialog: { - openMs: number; - }; - rawQueries: RawQueryResult[]; - typingSimulation: TypingSimulationResult; -}; - -type CliOptions = { - url: string; - outputPath?: string; - queriesPath?: string; - cadenceMs: number; - timeoutMs: number; - width: number; - height: number; - deviceScaleFactor: number; -}; - -function parseArgs(argv: string[]): CliOptions { - const options: CliOptions = { - url: "", - cadenceMs: 100, - timeoutMs: 10_000, - width: 1440, - height: 900, - deviceScaleFactor: 1, - }; - - for (let i = 0; i < argv.length; i += 1) { - const arg = argv[i]; - const next = argv[i + 1]; - - switch (arg) { - case "--url": - options.url = next ?? ""; - i += 1; - break; - case "--output": - options.outputPath = next; - i += 1; - break; - case "--queries": - options.queriesPath = next; - i += 1; - break; - case "--typing-cadence-ms": - options.cadenceMs = Number(next); - i += 1; - break; - case "--timeout-ms": - options.timeoutMs = Number(next); - i += 1; - break; - case "--width": - options.width = Number(next); - i += 1; - break; - case "--height": - options.height = Number(next); - i += 1; - break; - case "--device-scale-factor": - options.deviceScaleFactor = Number(next); - i += 1; - break; - default: - break; - } - } - - if (!options.url) { - throw new Error("missing required --url"); - } - - return options; -} - -function sanitizeFileSegment(value: string): string { - return value - .toLowerCase() - .replace(/^https?:\/\//, "") - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 80); -} - -function defaultQueries(url: URL): QueryCase[] { - if (url.pathname.startsWith("/docs")) { - return [ - { - name: "purchase-controller", - query: "purchase controller", - expectedText: "PurchaseController", - }, - { - name: "apple-search-ads", - query: "apple search ads", - expectedText: "Apple Search Ads", - }, - { - name: "refund-protection", - query: "refund protection", - expectedText: "Refund Protection", - }, - ]; - } - - return [ - { - name: "search", - query: "search", - expectedText: "Search", - }, - ]; -} - -function defaultTypingQuery(url: URL): QueryCase { - if (url.pathname.startsWith("/docs")) { - return { - name: "posthog", - query: "posthog", - expectedText: "PostHog", - }; - } - - return { - name: "typing-search", - query: "search", - expectedText: "Search", - }; -} - -async function loadQueries(options: CliOptions): Promise { - if (!options.queriesPath) { - return defaultQueries(new URL(options.url)); - } - - const file = Bun.file(options.queriesPath); - return (await file.json()) as QueryCase[]; -} - -async function runCommand(command: string[], cwd: string): Promise { - const proc = Bun.spawn({ - cmd: command, - cwd, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - env: process.env, - }); - - const [stdout, stderr, exitCode] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - - if (exitCode !== 0) { - throw new Error( - `command failed (${exitCode}): ${command.join(" ")}\n${stdout}\n${stderr}`.trim(), - ); - } - - return stdout.trim(); -} - -async function runAgentBrowser( - sessionName: string, - profileDir: string, - cwd: string, - args: string[], -): Promise { - return runCommand( - ["agent-browser", "--session-name", sessionName, "--profile", profileDir, ...args], - cwd, - ); -} - -async function sleep(ms: number): Promise { - await Bun.sleep(ms); -} - -async function waitFor( - fn: () => Promise, - predicate: (value: T) => boolean, - timeoutMs: number, - intervalMs = 50, - label = "condition", -): Promise { - const start = performance.now(); - - while (true) { - const value = await fn(); - if (predicate(value)) return value; - if (performance.now() - start > timeoutMs) { - throw new Error(`timed out waiting for ${label}`); - } - await sleep(intervalMs); - } -} - - -async function dialogIncludesText( - sessionName: string, - profileDir: string, - cwd: string, - text: string, -): Promise { - const snapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot"]); - return snapshot.includes(text); -} - -async function listSearchRequests( - sessionName: string, - profileDir: string, - cwd: string, -): Promise { - const output = await runAgentBrowser(sessionName, profileDir, cwd, [ - "network", - "requests", - "--type", - "fetch", - ]); - const observedAtMs = Date.now(); - const lines = output.split("\n"); - const requests: SearchRequestRecord[] = []; - - for (const line of lines) { - const match = line.match(/^\[([^\]]+)\]\s+([A-Z]+)\s+(https?:\/\/\S+)\s+\(Fetch\)(?:\s+(\d{3}))?$/); - if (!match) continue; - - const [, requestId, method, url, status] = match; - const parsedUrl = new URL(url); - let kind: SearchRequestRecord["kind"] | null = null; - - if (parsedUrl.pathname.endsWith("/api/search")) { - kind = "api"; - } else if (parsedUrl.pathname.endsWith("/search-index.json")) { - kind = "index"; - } - - if (!kind) continue; - - requests.push({ - requestId, - kind, - method, - url, - query: - kind === "api" - ? parsedUrl.searchParams.get("query") ?? parsedUrl.searchParams.get("q") - : null, - status: status ? Number(status) : undefined, - observedAtMs, - }); - } - - return requests; -} - -function startSearchRequestObserver( - sessionName: string, - profileDir: string, - cwd: string, - knownRequestIds: Set, - intervalMs = 100, -) { - let stopped = false; - const observed = new Map(); - - const task = (async () => { - while (!stopped) { - const requests = await listSearchRequests(sessionName, profileDir, cwd); - for (const request of requests) { - if (knownRequestIds.has(request.requestId)) continue; - if (observed.has(request.requestId)) continue; - observed.set(request.requestId, request); - } - - await sleep(intervalMs); - } - - return [...observed.values()].sort((left, right) => left.observedAtMs - right.observedAtMs); - })(); - - return { - async stop() { - stopped = true; - return task; - }, - }; -} - -async function benchmarkRawQuery( - sessionName: string, - profileDir: string, - cwd: string, - inputRef: string, - queryCase: QueryCase, - timeoutMs: number, -): Promise { - await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, ""]); - await sleep(250); - const knownRequestIds = new Set( - (await listSearchRequests(sessionName, profileDir, cwd)).map((request) => request.requestId), - ); - const requestObserver = startSearchRequestObserver(sessionName, profileDir, cwd, knownRequestIds); - const startedAt = Date.now(); - const fillStart = performance.now(); - await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, queryCase.query]); - const inputSetMs = Number((performance.now() - fillStart).toFixed(2)); - - let firstVisibleResultMs: number | null = null; - try { - if (queryCase.expectedText) { - await waitFor( - async () => { - const isVisible = await dialogIncludesText(sessionName, profileDir, cwd, queryCase.expectedText!); - if (isVisible && firstVisibleResultMs === null) { - firstVisibleResultMs = Number((Date.now() - startedAt).toFixed(2)); - } - return isVisible; - }, - Boolean, - timeoutMs, - ); - } else { - await sleep(1_000); - } - } finally { - await sleep(150); - } - - const transportRequests = await requestObserver.stop(); - const firstRequest = transportRequests[0]; - - return { - name: queryCase.name, - query: queryCase.query, - expectedText: queryCase.expectedText, - inputSetMs, - firstTransportRequestSeenMs: - firstRequest ? Number((firstRequest.observedAtMs - startedAt).toFixed(2)) : null, - firstVisibleResultMs, - transportRequestCount: transportRequests.length, - transportRequests: transportRequests.map((request) => ({ - kind: request.kind, - query: request.query, - url: request.url, - status: request.status, - seenOffsetMs: Number((request.observedAtMs - startedAt).toFixed(2)), - })), - }; -} - -async function benchmarkTypingSimulation( - sessionName: string, - profileDir: string, - cwd: string, - inputRef: string, - finalQuery: QueryCase, - cadenceMs: number, - timeoutMs: number, -): Promise { - await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, ""]); - await sleep(250); - const knownRequestIds = new Set( - (await listSearchRequests(sessionName, profileDir, cwd)).map((request) => request.requestId), - ); - const requestObserver = startSearchRequestObserver(sessionName, profileDir, cwd, knownRequestIds); - const startedAt = Date.now(); - let currentValue = ""; - - for (const char of finalQuery.query) { - currentValue = `${currentValue}${char}`; - await runAgentBrowser(sessionName, profileDir, cwd, ["fill", inputRef, currentValue]); - if (!currentValue.endsWith(char)) { - throw new Error("failed to update search input during typing simulation"); - } - await sleep(cadenceMs); - } - - let finalResultVisibleMs: number | null = null; - try { - if (finalQuery.expectedText) { - await waitFor( - async () => { - const isVisible = await dialogIncludesText( - sessionName, - profileDir, - cwd, - finalQuery.expectedText!, - ); - if (isVisible && finalResultVisibleMs === null) { - finalResultVisibleMs = Number((Date.now() - startedAt).toFixed(2)); - } - return isVisible; - }, - Boolean, - timeoutMs, - ); - } else { - await sleep(1_000); - } - } finally { - await sleep(150); - } - - const transportRequests = await requestObserver.stop(); - - const requests = transportRequests.map((request) => ({ - kind: request.kind, - query: request.query, - url: request.url, - status: request.status, - seenOffsetMs: Number((request.observedAtMs - startedAt).toFixed(2)), - })); - - const requestSeenOffsets = requests - .map((request) => request.seenOffsetMs) - .filter((value): value is number => value != null) - .sort((left, right) => left - right); - const totalWallMs = Number((Date.now() - startedAt).toFixed(2)); - - return { - query: finalQuery.query, - cadenceMs, - totalWallMs, - firstTransportRequestSeenMs: requestSeenOffsets[0] ?? null, - finalTransportRequestSeenMs: requestSeenOffsets.at(-1) ?? null, - finalResultVisibleMs, - transportRequests: requests, - }; -} - -function printSummary(result: BenchmarkResult) { - const lines = [ - `URL: ${result.url}`, - `Output: ${result.outputPath}`, - "", - `Initial load: ${result.navigation.openMs.toFixed(2)}ms`, - `Search button ready: ${result.navigation.searchButtonReadyMs.toFixed(2)}ms`, - `Search dialog open: ${result.searchDialog.openMs.toFixed(2)}ms`, - "", - "Raw queries:", - ]; - - for (const item of result.rawQueries) { - lines.push( - `- ${item.name}: visible=${item.firstVisibleResultMs ?? "n/a"}ms, transport=${item.transportRequestCount}`, - ); - } - - lines.push(""); - lines.push( - `Typing simulation (${result.typingSimulation.query}, cadence ${result.typingSimulation.cadenceMs}ms): visible=${result.typingSimulation.finalResultVisibleMs ?? "n/a"}ms, transport=${result.typingSimulation.transportRequests.length}`, - ); - - console.log(lines.join("\n")); -} - -function parseSnapshotRef(snapshot: string, role: "button" | "textbox", labelPattern: RegExp): string { - const lines = snapshot.split("\n"); - for (const line of lines) { - if (!line.includes(`- ${role} `)) continue; - if (!labelPattern.test(line)) continue; - const match = line.match(/\[ref=(e\d+)\]/); - if (match) return `@${match[1]}`; - } - - throw new Error(`could not find ${role} ref matching ${labelPattern}`); -} - -async function main() { - const cwd = process.cwd(); - const options = parseArgs(process.argv.slice(2)); - const url = new URL(options.url); - const queries = await loadQueries(options); - const typingQuery = defaultTypingQuery(url); - - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const sessionName = `search-benchmark-${Date.now().toString(36)}`; - const profileDir = path.join(cwd, ".context", "browser-profiles", sessionName); - const outputPath = - options.outputPath ?? - path.join( - cwd, - ".context", - `search-benchmark-${sanitizeFileSegment(`${url.host}${url.pathname}`)}-${timestamp}.json`, - ); - - await mkdir(path.dirname(outputPath), { recursive: true }); - await mkdir(profileDir, { recursive: true }); - - await runAgentBrowser(sessionName, profileDir, cwd, [ - "open", - "about:blank", - ]); - await runAgentBrowser(sessionName, profileDir, cwd, [ - "set", - "viewport", - String(options.width), - String(options.height), - String(options.deviceScaleFactor), - ]); - - try { - const navigationStart = performance.now(); - await runAgentBrowser(sessionName, profileDir, cwd, ["open", options.url]); - const openMs = Number((performance.now() - navigationStart).toFixed(2)); - - const metrics = null; - const userAgent = "agent-browser"; - const searchButtonReadyMs = openMs; - - const snapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot", "-i"]); - const searchButtonRef = parseSnapshotRef(snapshot, "button", /Search/i); - - const openDialogStart = performance.now(); - await runAgentBrowser(sessionName, profileDir, cwd, ["click", searchButtonRef]); - await waitFor( - async () => { - const latestSnapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot", "-i"]); - return latestSnapshot.includes('textbox "Search"'); - }, - Boolean, - options.timeoutMs, - 50, - "search dialog input", - ); - const searchDialogOpenMs = Number((performance.now() - openDialogStart).toFixed(2)); - const dialogSnapshot = await runAgentBrowser(sessionName, profileDir, cwd, ["snapshot", "-i"]); - const inputRef = parseSnapshotRef(dialogSnapshot, "textbox", /^- textbox "Search"/); - - const rawQueries: RawQueryResult[] = []; - for (const queryCase of queries) { - rawQueries.push( - await benchmarkRawQuery( - sessionName, - profileDir, - cwd, - inputRef, - queryCase, - options.timeoutMs, - ), - ); - } - - const typingSimulation = await benchmarkTypingSimulation( - sessionName, - profileDir, - cwd, - inputRef, - typingQuery, - options.cadenceMs, - options.timeoutMs, - ); - - const result: BenchmarkResult = { - url: options.url, - generatedAt: new Date().toISOString(), - outputPath, - sessionName, - userAgent, - viewport: { - width: options.width, - height: options.height, - deviceScaleFactor: options.deviceScaleFactor, - }, - navigation: { - openMs, - searchButtonReadyMs, - metrics, - }, - searchDialog: { - openMs: searchDialogOpenMs, - }, - rawQueries, - typingSimulation, - }; - - await writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`, "utf8"); - printSummary(result); - } finally { - await runAgentBrowser(sessionName, profileDir, cwd, ["close"]).catch(() => undefined); - } -} - -await main(); diff --git a/src/lib/search.shared.ts b/src/lib/search.shared.ts index f279ea0..f6ebf57 100644 --- a/src/lib/search.shared.ts +++ b/src/lib/search.shared.ts @@ -3,13 +3,13 @@ import { buildDocsPath } from "./url-base"; export const SEARCH_INDEX_FILENAME = "search-index.json"; export const SEARCH_INDEX_PATH = buildDocsPath(SEARCH_INDEX_FILENAME); -export const DASHBOARD_SEARCH_GROUP = "dashboard"; -export const WEB_CHECKOUT_SEARCH_GROUP = "web-checkout"; -export const INTEGRATIONS_SEARCH_GROUP = "integrations"; -export const SUPPORT_SEARCH_GROUP = "support"; -export const GENERAL_SEARCH_GROUP = "general"; -export const COMMUNITY_SEARCH_GROUP = "community"; -export const SDK_SEARCH_GROUPS = [ +const DASHBOARD_SEARCH_GROUP = "dashboard"; +const WEB_CHECKOUT_SEARCH_GROUP = "web-checkout"; +const INTEGRATIONS_SEARCH_GROUP = "integrations"; +const SUPPORT_SEARCH_GROUP = "support"; +const GENERAL_SEARCH_GROUP = "general"; +const COMMUNITY_SEARCH_GROUP = "community"; +const SDK_SEARCH_GROUPS = [ "ios", "android", "flutter", @@ -37,7 +37,7 @@ export const SEARCH_SCOPE_OPTIONS: Array<{ name: string; value: SearchScope }> = { name: "React Native", value: "react-native" }, ]; -export const COMMON_SEARCH_GROUPS: SearchCommonGroup[] = [ +const COMMON_SEARCH_GROUPS: SearchCommonGroup[] = [ DASHBOARD_SEARCH_GROUP, WEB_CHECKOUT_SEARCH_GROUP, INTEGRATIONS_SEARCH_GROUP, @@ -48,12 +48,9 @@ export const COMMON_SEARCH_GROUPS: SearchCommonGroup[] = [ const SDK_SEARCH_GROUP_SET = new Set(SDK_SEARCH_GROUPS); const COMMON_SEARCH_GROUP_SET = new Set(COMMON_SEARCH_GROUPS); -export function getSearchGroupsForScope(scope?: SearchScope): SearchGroup[] { - if (scope) { - return [scope, ...COMMON_SEARCH_GROUPS]; - } - - return [...COMMON_SEARCH_GROUPS, COMMUNITY_SEARCH_GROUP]; +export function getSearchGroupsForScope(scope?: SearchScope): SearchGroup[] | undefined { + if (!scope) return undefined; + return [scope, ...COMMON_SEARCH_GROUPS]; } export function getSearchGroupFromUrl(url: string): SearchGroup { diff --git a/src/lib/static-search-client.ts b/src/lib/static-search-client.ts index a5a7395..50f59e4 100644 --- a/src/lib/static-search-client.ts +++ b/src/lib/static-search-client.ts @@ -10,7 +10,6 @@ import { import type { SearchGroup } from "./search.shared"; const indexCache = new Map>(); -const INDEX_CACHE_NAME = "docs-static-search-index-v1"; type StaticSearchClientOptions = { from: string; @@ -18,23 +17,7 @@ type StaticSearchClientOptions = { }; async function fetchIndexPayload(from: string): Promise { - if (typeof window !== "undefined" && "caches" in window) { - const cache = await window.caches.open(INDEX_CACHE_NAME); - const cachedResponse = await cache.match(from); - if (cachedResponse) { - return cachedResponse.json() as Promise; - } - - const response = await fetch(from, { cache: "force-cache" }); - if (!response.ok) { - throw new Error(`failed to fetch search index from ${from}`); - } - - void cache.put(from, response.clone()); - return response.json() as Promise; - } - - const response = await fetch(from, { cache: "force-cache" }); + const response = await fetch(from, { cache: "no-cache" }); if (!response.ok) { throw new Error(`failed to fetch search index from ${from}`); } From 5b5fc727ded3e38b385600d0bc69fb4c530e4a34 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Wed, 15 Apr 2026 16:37:05 -0700 Subject: [PATCH 09/11] Move agentation to dev dependencies --- bun.lock | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bun.lock b/bun.lock index 8462609..104868f 100644 --- a/bun.lock +++ b/bun.lock @@ -12,7 +12,6 @@ "@tanstack/react-router": "1.163.3", "@tanstack/react-router-devtools": "1.163.3", "@tanstack/react-start": "1.166.1", - "agentation": "^3.0.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "flexsearch": "^0.8.212", @@ -47,6 +46,7 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.4", + "agentation": "^3.0.2", "next-validate-link": "^1.6.4", "oxfmt": "^0.36.0", "oxlint": "^1.51.0", diff --git a/package.json b/package.json index 4f7f2c4..a719127 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,6 @@ "@tanstack/react-router": "1.163.3", "@tanstack/react-router-devtools": "1.163.3", "@tanstack/react-start": "1.166.1", - "agentation": "^3.0.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "flexsearch": "^0.8.212", @@ -71,6 +70,7 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.4", + "agentation": "^3.0.2", "next-validate-link": "^1.6.4", "oxfmt": "^0.36.0", "oxlint": "^1.51.0", From 1b23c07bcbd3bfaa43e1bda29e631f1d4a496f18 Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Wed, 15 Apr 2026 18:38:31 -0700 Subject: [PATCH 10/11] Fix scoped static search retrieval --- src/lib/search-index.test.ts | 34 ++++++++++++++++---------- src/lib/search-index.ts | 46 +++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/src/lib/search-index.test.ts b/src/lib/search-index.test.ts index f19c0ea..f294cb7 100644 --- a/src/lib/search-index.test.ts +++ b/src/lib/search-index.test.ts @@ -9,29 +9,29 @@ import { const documents: StaticSearchDocument[] = [ { - id: "/docs/ios/purchase-controller", - page_id: "/docs/ios/purchase-controller", + id: "/docs/ios/sdk-reference/PurchaseController", + page_id: "/docs/ios/sdk-reference/PurchaseController", type: "page", - content: "Purchase Controller", + content: "PurchaseController", breadcrumbs: ["Docs", "iOS"], tags: ["ios"], - url: "/docs/ios/purchase-controller", + url: "/docs/ios/sdk-reference/PurchaseController", }, { - id: "/docs/ios/purchase-controller-0", - page_id: "/docs/ios/purchase-controller", + id: "/docs/ios/sdk-reference/PurchaseController-0", + page_id: "/docs/ios/sdk-reference/PurchaseController", type: "heading", content: "PurchaseController", tags: ["ios"], - url: "/docs/ios/purchase-controller#purchase-controller", + url: "/docs/ios/sdk-reference/PurchaseController#purchase-controller", }, { - id: "/docs/ios/purchase-controller-1", - page_id: "/docs/ios/purchase-controller", + id: "/docs/ios/sdk-reference/PurchaseController-1", + page_id: "/docs/ios/sdk-reference/PurchaseController", type: "text", content: "Use PurchaseController to manage purchases.", tags: ["ios"], - url: "/docs/ios/purchase-controller#purchase-controller", + url: "/docs/ios/sdk-reference/PurchaseController#purchase-controller", }, { id: "/docs/android/sdk-reference/PurchaseController", @@ -79,9 +79,9 @@ describe("static search index", () => { expect(exported.raw).not.toEqual({}); expect(results.length).toBeGreaterThan(0); expect(results[0]).toMatchObject({ - id: "/docs/ios/purchase-controller", + id: "/docs/ios/sdk-reference/PurchaseController", breadcrumbs: ["Docs", "iOS"], - url: "/docs/ios/purchase-controller", + url: "/docs/ios/sdk-reference/PurchaseController", }); }); @@ -102,4 +102,14 @@ describe("static search index", () => { url: "/docs/android/sdk-reference/PurchaseController", }); }); + + test("supports scoped prefix queries under strict tokenization", async () => { + const index = buildStaticSearchIndex(documents); + const results = await searchStaticSearchIndex(index, "purchase con", "ios"); + + expect(results[0]).toMatchObject({ + id: "/docs/ios/sdk-reference/PurchaseController", + url: "/docs/ios/sdk-reference/PurchaseController", + }); + }); }); diff --git a/src/lib/search-index.ts b/src/lib/search-index.ts index ba7b430..d8039ce 100644 --- a/src/lib/search-index.ts +++ b/src/lib/search-index.ts @@ -33,18 +33,37 @@ function normalizeTag(tag?: string | string[]): string[] { return Array.isArray(tag) ? tag : [tag]; } -function expandSearchContent(value: string): string { +function getPrefixTokens(value: string): string[] { + const prefixes = new Set(); + const terms = tokenizeForMatch(value); + + for (const term of terms) { + for (let length = 3; length < term.length; length += 1) { + prefixes.add(term.slice(0, length)); + } + } + + return [...prefixes]; +} + +function expandSearchContent(document: StaticSearchDocument): string { + const value = document.content; const normalized = value .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/[_/.-]+/g, " ") .trim() .replace(/\s+/g, " "); - if (!normalized || normalized === value) { - return value; + const parts = [value]; + if (normalized && normalized !== value) { + parts.push(normalized); + } + + if (document.type !== "text") { + parts.push(...getPrefixTokens(normalized || value)); } - return `${value}\n${normalized}`; + return [...new Set(parts)].join("\n"); } function createSearchDocument() { @@ -55,8 +74,7 @@ function createSearchDocument() { document: { id: "id", index: ["search_content"], - tag: ["tags"], - store: ["id", "page_id", "type", "content", "breadcrumbs", "url"], + store: ["id", "page_id", "type", "content", "breadcrumbs", "tags", "url"], }, }); } @@ -66,7 +84,7 @@ export function buildStaticSearchIndex(documents: StaticSearchDocument[]): Stati for (const document of documents) { index.add(document.id, { ...document, - search_content: document.search_content ?? expandSearchContent(document.content), + search_content: document.search_content ?? expandSearchContent(document), }); } return index; @@ -89,6 +107,10 @@ function tokenizeForMatch(value: string): string[] { return normalized ? normalized.split(" ") : []; } +function matchesTag(doc: StaticSearchDocument, tags: string[]): boolean { + return tags.length === 0 || doc.tags.some((tag) => tags.includes(tag)); +} + function scorePrefixCoverage(queryTerms: string[], candidateTerms: string[]): number { if (queryTerms.length === 0 || candidateTerms.length === 0) return 0; if (!queryTerms.every((term) => candidateTerms.some((candidate) => candidate.startsWith(term)))) { @@ -109,6 +131,8 @@ function scoreUrlMatch(query: string, url: string): number { if (!normalizedSegment) return 0; if (compactSegment === compactQuery) return 900; if (normalizedSegment === normalizedQuery) return 850; + if (compactSegment.startsWith(compactQuery)) return 800; + if (normalizedSegment.startsWith(normalizedQuery)) return 750; if (compactSegment.includes(compactQuery)) return 500; if (normalizedSegment.includes(normalizedQuery)) return 450; return 0; @@ -135,6 +159,8 @@ function scoreGroupedPage(query: string, page: StaticSearchDocument, items: Stat } else if (normalizedTitle === normalizedQuery) { score += 3_500; } else { + if (compactTitle.startsWith(compactQuery)) score += 2_600; + if (normalizedTitle.startsWith(normalizedQuery)) score += 2_200; if (compactTitle.includes(compactQuery)) score += 1_500; if (normalizedTitle.includes(normalizedQuery)) score += 1_200; } @@ -160,6 +186,8 @@ function scoreGroupedPage(query: string, page: StaticSearchDocument, items: Stat if (compactHeading.includes(compactQuery)) score += 750; if (normalizedHeading.includes(normalizedQuery)) score += 650; + if (compactHeading.startsWith(compactQuery)) score += 550; + if (normalizedHeading.startsWith(normalizedQuery)) score += 450; score += scorePrefixCoverage(queryTerms, headingTerms); } @@ -206,11 +234,10 @@ export async function searchStaticSearchIndex( limit = 60, ): Promise { const tags = normalizeTag(tag); - const rawLimit = Math.max(limit * 4, 120); + const rawLimit = tags.length > 0 ? Math.max(limit * 40, 2_000) : Math.max(limit * 8, 500); const groups = await index.searchAsync(query, { index: "search_content", limit: rawLimit, - tag: tags.length > 0 ? { tags } : undefined, }); if (groups.length === 0) return []; @@ -222,6 +249,7 @@ export async function searchStaticSearchIndex( for (const [rank, id] of results.entries()) { const doc = index.get(id) as StaticSearchDocument | null; if (!doc) continue; + if (!matchesTag(doc, tags)) continue; const entry = grouped.get(doc.page_id) ?? { firstRank: rank, From bebda95d47e14c4cbf3d1cede3af584a6c9fc58d Mon Sep 17 00:00:00 2001 From: Duncan Crawbuck Date: Wed, 15 Apr 2026 19:23:16 -0700 Subject: [PATCH 11/11] Harden static search scoring and export --- src/lib/search-index.ts | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/lib/search-index.ts b/src/lib/search-index.ts index d8039ce..b711b43 100644 --- a/src/lib/search-index.ts +++ b/src/lib/search-index.ts @@ -14,7 +14,7 @@ export type StaticSearchDocument = { export type StaticSearchExport = { type: "default"; - raw: Record; + raw: Record; }; export type StaticSearchResult = { @@ -159,10 +159,15 @@ function scoreGroupedPage(query: string, page: StaticSearchDocument, items: Stat } else if (normalizedTitle === normalizedQuery) { score += 3_500; } else { - if (compactTitle.startsWith(compactQuery)) score += 2_600; - if (normalizedTitle.startsWith(normalizedQuery)) score += 2_200; - if (compactTitle.includes(compactQuery)) score += 1_500; - if (normalizedTitle.includes(normalizedQuery)) score += 1_200; + if (compactTitle.startsWith(compactQuery)) { + score += 2_600; + } else if (normalizedTitle.startsWith(normalizedQuery)) { + score += 2_200; + } else if (compactTitle.includes(compactQuery)) { + score += 1_500; + } else if (normalizedTitle.includes(normalizedQuery)) { + score += 1_200; + } } score += scorePrefixCoverage(queryTerms, titleTerms); @@ -184,10 +189,15 @@ function scoreGroupedPage(query: string, page: StaticSearchDocument, items: Stat continue; } - if (compactHeading.includes(compactQuery)) score += 750; - if (normalizedHeading.includes(normalizedQuery)) score += 650; - if (compactHeading.startsWith(compactQuery)) score += 550; - if (normalizedHeading.startsWith(normalizedQuery)) score += 450; + if (compactHeading.startsWith(compactQuery)) { + score += 1_100; + } else if (normalizedHeading.startsWith(normalizedQuery)) { + score += 900; + } else if (compactHeading.includes(compactQuery)) { + score += 750; + } else if (normalizedHeading.includes(normalizedQuery)) { + score += 650; + } score += scorePrefixCoverage(queryTerms, headingTerms); } @@ -203,9 +213,9 @@ function scoreGroupedPage(query: string, page: StaticSearchDocument, items: Stat } export async function exportStaticSearchIndex(index: StaticSearchIndex): Promise { - const raw: Record = {}; + const raw: StaticSearchExport["raw"] = {}; const maybePromise = index.export((key, value) => { - raw[key] = value; + raw[key] = value ?? null; }); await maybePromise; return { @@ -218,7 +228,7 @@ export async function importStaticSearchIndex(data: StaticSearchExport): Promise const index = createSearchDocument(); const imports: Promise[] = []; for (const [key, value] of Object.entries(data.raw)) { - const maybePromise = index.import(key, value); + const maybePromise = index.import(key, value ?? undefined); if (maybePromise?.then) { imports.push(maybePromise); }