From 53aff07d0c72702fe19f3ff49651b18b3df5b05b Mon Sep 17 00:00:00 2001 From: Ross Date: Thu, 12 Mar 2026 15:30:33 -0400 Subject: [PATCH] fix missing html on ai/raw route --- README.md | 2 +- docs/getting-started/llm-friendly-docs.md | 2 + scripts/fetchedAddressData.json | 2 +- scripts/generateAiArtifacts.ts | 93 ++++++++++++++++++++++- 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index af0976e19..f5c14ed5f 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ The build also generates AI-friendly artifacts that will be served by the hosted - `build/llms.txt` (served as `/llms.txt`) - `build/ai/manifest.json` (served as `/ai/manifest.json`) - `build/ai/docs.jsonl` (served as `/ai/docs.jsonl`) -- `build/ai/raw/` (served as `/ai/raw/`) +- `build/ai/raw/` (served as `/ai/raw/`, with an index page plus raw `.md`/`.mdx` files) Artifact URLs default to `https://docs.yearn.fi`. To generate artifacts against a preview deployment, set `DOCS_URL` explicitly (for example `DOCS_URL=https://`). diff --git a/docs/getting-started/llm-friendly-docs.md b/docs/getting-started/llm-friendly-docs.md index 0b515be40..6cb15430a 100644 --- a/docs/getting-started/llm-friendly-docs.md +++ b/docs/getting-started/llm-friendly-docs.md @@ -9,6 +9,8 @@ This page makes it easy to share Yearn docs with an LLM using machine-readable e - https://docs.yearn.fi/ai/docs.jsonl - https://docs.yearn.fi/ai/raw/ +`/ai/raw/` is the landing page for the raw markdown mirror. Use the exact `source.rawPath` values from `docs.jsonl` when you need a specific `.md` or `.mdx` source file. + ## Example prompt ```text diff --git a/scripts/fetchedAddressData.json b/scripts/fetchedAddressData.json index 1a7db75f4..bb4271299 100644 --- a/scripts/fetchedAddressData.json +++ b/scripts/fetchedAddressData.json @@ -1,5 +1,5 @@ { - "timeLastChecked": 1773280154, + "timeLastChecked": 1773343605, "addressesData": { "v3ContractAddresses": { "topLevel": { diff --git a/scripts/generateAiArtifacts.ts b/scripts/generateAiArtifacts.ts index 08fae9b25..3a1bbd4a9 100644 --- a/scripts/generateAiArtifacts.ts +++ b/scripts/generateAiArtifacts.ts @@ -71,6 +71,15 @@ function normalizeText(input: string) { .trim() } +function escapeHtml(input: string) { + return input + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + function extractAttr(tagAttrs: string, name: string) { const re = new RegExp( `${name}=(?:"([^"]+)"|'([^']+)'|([^\\s>]+))`, @@ -292,6 +301,79 @@ function findRawSourceRelativePath(docsDir: string, relHtmlPath: string) { return undefined } +function writeRawDocsIndex(rawOutDir: string, rawFiles: string[]) { + const fileList = rawFiles + .slice() + .sort((a, b) => a.localeCompare(b)) + .map((relPath) => { + const href = encodeURI(relPath.replace(/\\/g, '/')) + const label = escapeHtml(relPath.replace(/\\/g, '/')) + return `
  • ${label}
    • ` + }) + .join('\n') + + const html = [ + '', + '', + ' ', + ' ', + ' ', + ' Yearn Docs Raw Source Mirror', + ' ', + ' ', + ' ', + '
    ', + '

    Yearn Docs Raw Source Mirror

    ', + '

    This endpoint mirrors the repository docs/ tree as raw .md and .mdx files.

    ', + '

    Use /ai/docs.jsonl for retrieval, then follow each record’s source.rawPath to fetch the exact source file when you need original markdown or MDX.

    ', + `

    ${rawFiles.length} raw source files are available below.

    `, + '
      ', + fileList, + '
    ', + '
    ', + ' ', + '', + '', + ].join('\n') + + fs.writeFileSync(path.join(rawOutDir, 'index.html'), html) +} + function main() { const workspaceRoot = process.cwd() const buildDir = path.join(workspaceRoot, 'build') @@ -372,6 +454,13 @@ function main() { docsJsonlStream.end() copyRawDocs(docsDir, rawOutDir) + const rawFiles = walkFiles(rawOutDir) + .filter((p) => { + const ext = path.extname(p).toLowerCase() + return ext === '.md' || ext === '.mdx' + }) + .map((p) => path.relative(rawOutDir, p)) + writeRawDocsIndex(rawOutDir, rawFiles) const manifest: ManifestV1 = { schemaVersion: 2, @@ -396,11 +485,11 @@ function main() { 'AI-readable exports:', '- Manifest: /ai/manifest.json', '- Plaintext corpus (JSONL): /ai/docs.jsonl', - '- Raw docs sources (MD/MDX): /ai/raw/ (mirrors repository `docs/`)', + '- Raw docs source mirror + index: /ai/raw/ (mirrors repository `docs/`)', '', 'Notes:', '- Prefer citing canonical page URLs on https://docs.yearn.fi', - '- Use plaintext for retrieval; fall back to raw MDX/MD for exact formatting/quotes', + '- Use plaintext for retrieval; follow `source.rawPath` for exact raw MDX/MD files', '', ].join('\n') )