diff --git a/layouts/robots.txt b/layouts/robots.txt index 28b3cbc..e5a1532 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -1,5 +1,12 @@ User-agent: * +# 不允许抓取 Cloudflare 的内部目录 Disallow: /cdn-cgi/ -Sitemap: https://ai.hubtoday.app/sitemap.xml \ No newline at end of file +# 新增:不允许抓取 /en 和 /ja 目录 +Disallow: /en/ +Disallow: /ja/ + +# 站点地图 +Sitemap: https://ai.hubtoday.app/sitemap.xml +Sitemap: https://ai.hubtoday.app/zh/sitemap.xml