Update robots.txt

2025-08-12 21:35:54 +08:00
parent b6790a1c37
commit 2330ead861
1 changed files with 8 additions and 1 deletions
--- a/layouts/robots.txt
+++ b/layouts/robots.txt
@@ -1,5 +1,12 @@
 User-agent: *

+# 不允许抓取 Cloudflare 的内部目录
 Disallow: /cdn-cgi/

-Sitemap: https://ai.hubtoday.app/sitemap.xml
+# 新增：不允许抓取 /en 和 /ja 目录
+Disallow: /en/
+Disallow: /ja/
+
+# 站点地图
+Sitemap: https://ai.hubtoday.app/sitemap.xml
+Sitemap: https://ai.hubtoday.app/zh/sitemap.xml