DOC-3385: Markdown for Agents - build-time HTML to Markdown conversion

.github/workflows/deploy_docs_v2.yml

@@ -53,6 +53,9 @@ jobs:
     - name: Copy llms.txt files to root
       run: ./-scripts/copy-llms-files.sh build/site
 
+    - name: Generate markdown for agents
+      run: yarn build:markdown build/site
+
     - name: Rename site folder to docs
       run: |
         mv ./build/site ./build/docs

.gitignore

@@ -13,3 +13,9 @@ vendor
 
 # Antora tmp files
 _site/
+
+# Cursor setup (local only, do not push)
+.cursor/
+AGENTS.md
+.cursorignore
+context7.json

README.md

@@ -1,6 +1,8 @@
 # TinyMCE Documentation
 
-This project maintains the official documentation for TinyMCE, available at [https://www.tiny.cloud/docs](https://www.tiny.cloud/docs). If you have modifications or improvements to contribute, fork this repository, make the necessary changes, and submit a pull request (PR). A contributor's license agreement (CLA) must be signed before your contribution can be merged. This agreement will be sent via email when you create a PR.
+This project maintains the official documentation for TinyMCE, available at [https://www.tiny.cloud/docs](https://www.tiny.cloud/docs). **TinyMCE 8 is the current stable version.** Use `tinymce@8` or `tinymce/8` in CDN URLs and package installations for new projects.
+
+If you have modifications or improvements to contribute, fork this repository, make the necessary changes, and submit a pull request (PR). A contributor's license agreement (CLA) must be signed before your contribution can be merged. This agreement will be sent via email when you create a PR.
 
 This project is built using [Antora](https://antora.org/)
 
@@ -99,3 +101,16 @@ To help with this, there are two mechanisms for overriding the `tinymce.min.js`
    ```
 
 > **Caution:** Use this sparingly. Avoid using different URLs for multiple demos on the same page, and remember to revert these changes once the feature is fully released.
+
+## Resources for AI and LLM Consumers
+
+The published documentation site provides several artifacts optimized for AI agents and LLM-based tools:
+
+| Resource | URL | Description |
+|----------|-----|--------------|
+| `llms.txt` | [https://www.tiny.cloud/docs/llms.txt](https://www.tiny.cloud/docs/llms.txt) | Concise overview with key links and code examples |
+| `llms-full.txt` | [https://www.tiny.cloud/docs/llms-full.txt](https://www.tiny.cloud/docs/llms-full.txt) | Full index of documentation pages with URLs |
+
+**Version guidance:** Always prefer TinyMCE 8 for new projects. The `/tinymce/latest/` URL segment serves TinyMCE 8 content. Use `tinymce@8` or `tinymce/8` in CDN URLs and package installations.
+
+In addition, each HTML page has a Markdown sibling (`.md`) for content negotiation. Clients that send `Accept: text/markdown` can request plain markdown instead of HTML for lower-token consumption. These markdown files are generated during deployment and are not produced by local builds or PR previews.

package.json

@@ -9,6 +9,9 @@
   "homepage": "https://www.tiny.cloud/docs/",
   "scripts": {
     "build-local-ref": "./-scripts/api-reference-local.sh",
+    "build:production": "antora ./antora-playbook.yml",
+    "build:markdown": "node scripts/generate-markdown.mjs",
+    "build:all": "yarn build:production && yarn build:markdown",
     "clean": "rm -rf ./build",
     "nodemon-dev": "nodemon --exec yarn antora ./antora-playbook.yml",
     "server": "http-server build/site/ --port 4000",
@@ -32,9 +35,12 @@
     "@antora/site-generator-default": "^3.1.10",
     "@tinymce/antora-extension-livedemos": "^0.1.0",
     "@tinymce/moxiedoc": "^0.3.0",
+    "dom-to-semantic-markdown": "^1.5.0",
     "dotenv": "^16.5.0",
     "ecstatic": "^4.1.4",
+    "gpt-tokenizer": "^3.4.0",
     "http-server": "^0.12.3",
+    "jsdom": "^24.1.0",
     "nodemon": "^3.1.10",
     "npm-run-all": "^4.1.5"
   }

scripts/generate-markdown.mjs

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+
+/**
+ * Post-build script: converts every Antora HTML page into a Markdown sibling
+ * so AI agents can fetch clean, low-token content via content negotiation.
+ *
+ * Uses dom-to-semantic-markdown (d2m) for conversion (preserves links in tables).
+ *
+ * Usage:  node scripts/generate-markdown.mjs [buildDir]
+ *         Default buildDir = build/site
+ */
+
+import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join, relative, dirname } from 'node:path';
+import { JSDOM } from 'jsdom';
+import { convertHtmlToMarkdown } from 'dom-to-semantic-markdown';
+import { encode } from 'gpt-tokenizer';
+
+const BUILD_DIR = process.argv[2] || 'build/site';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+async function* walkHtml(dir) {
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      yield* walkHtml(full);
+    } else if (entry.name.endsWith('.html')) {
+      yield full;
+    }
+  }
+}
+
+function extractTitle(doc) {
+  const h1 = doc.querySelector('article.doc h1');
+  if (h1) return h1.textContent.trim();
+  const title = doc.querySelector('title');
+  if (title) return title.textContent.trim().replace(/ \|.*$/, '');
+  return 'Untitled';
+}
+
+function buildFrontmatter(title, tokens) {
+  return [
+    '---',
+    `title: "${title.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`,
+    `tokens: ${tokens}`,
+    '---',
+    '',
+  ].join('\n');
+}
+
+/**
+ * Convert article.doc HTML to Markdown using dom-to-semantic-markdown.
+ */
+function convertToMarkdown(articleHtml, dom) {
+  return convertHtmlToMarkdown(articleHtml, {
+    overrideDOMParser: new dom.window.DOMParser(),
+    extractMainContent: false, // we already extracted article.doc
+    enableTableColumnTracking: false,
+    refifyUrls: false,
+    websiteDomain: 'https://www.tiny.cloud',
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+  const manifest = {};
+  let converted = 0;
+  let skipped = 0;
+
+  console.log(`Generating markdown siblings in ${BUILD_DIR} …`);
+
+  for await (const htmlPath of walkHtml(BUILD_DIR)) {
+    const html = await readFile(htmlPath, 'utf-8');
+    const dom = new JSDOM(html);
+    const article = dom.window.document.querySelector('article.doc');
+
+    if (!article) {
+      skipped++;
+      continue;
+    }
+
+    const title = extractTitle(dom.window.document);
+    const markdown = convertToMarkdown(article.innerHTML, dom);
+    const tokens = encode(markdown).length;
+    const frontmatter = buildFrontmatter(title, tokens);
+    const fullMd = frontmatter + markdown + '\n';
+
+    const mdPath = htmlPath.replace(/\.html$/, '.md');
+    await mkdir(dirname(mdPath), { recursive: true });
+    await writeFile(mdPath, fullMd, 'utf-8');
+
+    const urlPath = '/' + relative(BUILD_DIR, dirname(htmlPath)) + '/';
+    manifest[urlPath] = tokens;
+    converted++;
+  }
+
+  const manifestPath = join(BUILD_DIR, '_markdown-manifest.json');
+  await writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
+
+  console.log(
+    `Done. ${converted} pages converted, ${skipped} skipped (no article.doc). Manifest → ${manifestPath}`
+  );
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});