fix(seo): reduce sitemap bloat by filtering versioned docs and low-value pages

scripts/update-sitemap-loc.js

@@ -15,6 +15,53 @@ const sitemapXMLs = [
   ],
 ];
 
+/**
+ * URL patterns to exclude from the sitemap.
+ *
+ * Why:
+ * - Versioned doc URLs (e.g. /docs/apisix/3.14/) duplicate the latest
+ *   unversioned paths (e.g. /docs/apisix/) and bloat the sitemap.
+ *   Only the unversioned (latest) URLs should be indexed.
+ * - /docs/.../next/ pages are for unreleased development docs.
+ * - /search pages are blocked by robots.txt — keeping them in
+ *   the sitemap sends contradictory signals to crawlers.
+ * - /blog/tags/ and /blog/page/ are low-value aggregation/pagination
+ *   pages, also blocked by robots.txt.
+ */
+const excludePatterns = [
+  // Versioned docs: /docs/<project>/<version>/ where version is digits.digits
+  /\/docs\/[\w-]+\/\d+\.\d+\//,
+  // Development "next" docs
+  /\/docs\/[\w-]+\/next\//,
+  // Search pages (blocked by robots.txt)
+  /\/search\/?$/,
+  // Blog tag and pagination pages (blocked by robots.txt)
+  /\/blog\/tags\//,
+  /\/blog\/page\//,
+];
+
+/**
+ * Returns true if the URL should be excluded from the sitemap.
+ */
+function shouldExclude(url) {
+  return excludePatterns.some((pattern) => pattern.test(url));
+}
+
+/**
+ * Filter out excluded URLs from a sitemap object and return removal count.
+ */
+function filterSitemapUrls(sitemap) {
+  const urls = Array.isArray(sitemap.urlset.url)
+    ? sitemap.urlset.url
+    : [sitemap.urlset.url];
+  const before = urls.length;
+  sitemap.urlset.url = urls.filter((entry) => {
+    const loc = entry.loc && entry.loc._text;
+    return !loc || !shouldExclude(loc);
+  });
+  return before - sitemap.urlset.url.length;
+}
+
 const tasks = new Listr([
   {
     title: `Check sitemap.xml files exist`,
@@ -27,7 +74,7 @@ const tasks = new Listr([
     ),
   },
   {
-    title: `Merge sitemap.xml files`,
+    title: `Merge and filter sitemap.xml files`,
     task: () => new Listr(
       sitemapXMLs.map((group) => ({
         title: `Merge ${group[0]}`,
@@ -42,6 +89,8 @@ const tasks = new Listr([
                 ...sitemaps[i].urlset.url,
               ];
             }
+            const removed = filterSitemapUrls(res);
+            console.log(`  Filtered out ${removed} URLs from ${group[0]}`);
             return res;
           })
           .then((sitemap) => writeFile(group[0], js2xml(sitemap, { compact: true }, 'utf-8'))),

website/static/robots.txt

@@ -2,6 +2,119 @@
 
 User-agent: *
 
+# Blog aggregation and pagination pages (low-value for indexing)
+Disallow: /blog/tags/
+Disallow: /zh/blog/tags/
+Disallow: /blog/page/
+Disallow: /zh/blog/page/
+
+# Search pages
+Disallow: /search
+Disallow: /zh/search
+
+# Versioned docs — only the unversioned (latest) paths should be indexed.
+# e.g. /docs/apisix/ is the latest; /docs/apisix/3.14/ is a duplicate.
+Disallow: /docs/apisix/3.10/
+Disallow: /docs/apisix/3.11/
+Disallow: /docs/apisix/3.12/
+Disallow: /docs/apisix/3.13/
+Disallow: /docs/apisix/3.14/
+Disallow: /docs/apisix/3.15/
+Disallow: /docs/apisix/next/
+Disallow: /docs/ingress-controller/3.10/
+Disallow: /docs/ingress-controller/3.11/
+Disallow: /docs/ingress-controller/3.12/
+Disallow: /docs/ingress-controller/3.13/
+Disallow: /docs/ingress-controller/3.14/
+Disallow: /docs/ingress-controller/3.15/
+Disallow: /docs/ingress-controller/next/
+Disallow: /docs/helm-chart/3.10/
+Disallow: /docs/helm-chart/3.11/
+Disallow: /docs/helm-chart/3.12/
+Disallow: /docs/helm-chart/3.13/
+Disallow: /docs/helm-chart/3.14/
+Disallow: /docs/helm-chart/3.15/
+Disallow: /docs/helm-chart/next/
+Disallow: /docs/docker/3.10/
+Disallow: /docs/docker/3.11/
+Disallow: /docs/docker/3.12/
+Disallow: /docs/docker/3.13/
+Disallow: /docs/docker/3.14/
+Disallow: /docs/docker/3.15/
+Disallow: /docs/docker/next/
+Disallow: /docs/java-plugin-runner/3.10/
+Disallow: /docs/java-plugin-runner/3.11/
+Disallow: /docs/java-plugin-runner/3.12/
+Disallow: /docs/java-plugin-runner/3.13/
+Disallow: /docs/java-plugin-runner/3.14/
+Disallow: /docs/java-plugin-runner/3.15/
+Disallow: /docs/java-plugin-runner/next/
+Disallow: /docs/go-plugin-runner/3.10/
+Disallow: /docs/go-plugin-runner/3.11/
+Disallow: /docs/go-plugin-runner/3.12/
+Disallow: /docs/go-plugin-runner/3.13/
+Disallow: /docs/go-plugin-runner/3.14/
+Disallow: /docs/go-plugin-runner/3.15/
+Disallow: /docs/go-plugin-runner/next/
+Disallow: /docs/python-plugin-runner/3.10/
+Disallow: /docs/python-plugin-runner/3.11/
+Disallow: /docs/python-plugin-runner/3.12/
+Disallow: /docs/python-plugin-runner/3.13/
+Disallow: /docs/python-plugin-runner/3.14/
+Disallow: /docs/python-plugin-runner/3.15/
+Disallow: /docs/python-plugin-runner/next/
+
+# Chinese equivalents
+Disallow: /zh/docs/apisix/3.10/
+Disallow: /zh/docs/apisix/3.11/
+Disallow: /zh/docs/apisix/3.12/
+Disallow: /zh/docs/apisix/3.13/
+Disallow: /zh/docs/apisix/3.14/
+Disallow: /zh/docs/apisix/3.15/
+Disallow: /zh/docs/apisix/next/
+Disallow: /zh/docs/ingress-controller/3.10/
+Disallow: /zh/docs/ingress-controller/3.11/
+Disallow: /zh/docs/ingress-controller/3.12/
+Disallow: /zh/docs/ingress-controller/3.13/
+Disallow: /zh/docs/ingress-controller/3.14/
+Disallow: /zh/docs/ingress-controller/3.15/
+Disallow: /zh/docs/ingress-controller/next/
+Disallow: /zh/docs/helm-chart/3.10/
+Disallow: /zh/docs/helm-chart/3.11/
+Disallow: /zh/docs/helm-chart/3.12/
+Disallow: /zh/docs/helm-chart/3.13/
+Disallow: /zh/docs/helm-chart/3.14/
+Disallow: /zh/docs/helm-chart/3.15/
+Disallow: /zh/docs/helm-chart/next/
+Disallow: /zh/docs/docker/3.10/
+Disallow: /zh/docs/docker/3.11/
+Disallow: /zh/docs/docker/3.12/
+Disallow: /zh/docs/docker/3.13/
+Disallow: /zh/docs/docker/3.14/
+Disallow: /zh/docs/docker/3.15/
+Disallow: /zh/docs/docker/next/
+Disallow: /zh/docs/java-plugin-runner/3.10/
+Disallow: /zh/docs/java-plugin-runner/3.11/
+Disallow: /zh/docs/java-plugin-runner/3.12/
+Disallow: /zh/docs/java-plugin-runner/3.13/
+Disallow: /zh/docs/java-plugin-runner/3.14/
+Disallow: /zh/docs/java-plugin-runner/3.15/
+Disallow: /zh/docs/java-plugin-runner/next/
+Disallow: /zh/docs/go-plugin-runner/3.10/
+Disallow: /zh/docs/go-plugin-runner/3.11/
+Disallow: /zh/docs/go-plugin-runner/3.12/
+Disallow: /zh/docs/go-plugin-runner/3.13/
+Disallow: /zh/docs/go-plugin-runner/3.14/
+Disallow: /zh/docs/go-plugin-runner/3.15/
+Disallow: /zh/docs/go-plugin-runner/next/
+Disallow: /zh/docs/python-plugin-runner/3.10/
+Disallow: /zh/docs/python-plugin-runner/3.11/
+Disallow: /zh/docs/python-plugin-runner/3.12/
+Disallow: /zh/docs/python-plugin-runner/3.13/
+Disallow: /zh/docs/python-plugin-runner/3.14/
+Disallow: /zh/docs/python-plugin-runner/3.15/
+Disallow: /zh/docs/python-plugin-runner/next/
+
 Sitemap: https://apisix.apache.org/sitemap.xml
 
 Sitemap: https://apisix.apache.org/zh/sitemap.xml