[{"data":1,"prerenderedAt":168},["ShallowReactive",2],{"blog-post-robots-txt-guide-zh":3},{"id":4,"title":5,"body":6,"description":151,"extension":152,"meta":153,"navigation":163,"path":164,"seo":165,"stem":166,"__hash__":167},"blog\u002Fblog\u002Fzh\u002Frobots-txt-guide.md","Robots.txt 完全指南：如何控制搜索引擎抓取",{"type":7,"value":8,"toc":143},"minimark",[9,13,23,27,57,80,83,86,89,96,99,106,109,116,119,126,129,139],[10,11,5],"h1",{"id":12},"robotstxt-完全指南如何控制搜索引擎抓取",[14,15,16,17,22],"p",{},"Robots.txt 告诉搜索引擎爬虫哪些页面应该访问、哪些应该跳过。正确配置的 robots.txt 可以保护抓取预算，防止低价值页面被索引。使用我们的 ",[18,19,21],"a",{"href":20},"\u002Fzh\u002Ftools\u002Frobots-txt-generator","Robots.txt 生成器"," 快速创建。",[24,25,26],"h2",{"id":26},"基础语法",[28,29,34],"pre",{"className":30,"code":31,"language":32,"meta":33,"style":33},"language-txt shiki shiki-themes github-light github-dark","User-agent: *\nDisallow: \u002Fadmin\u002F\nSitemap: https:\u002F\u002Fexample.com\u002Fsitemap.xml\n","txt","",[35,36,37,45,51],"code",{"__ignoreMap":33},[38,39,42],"span",{"class":40,"line":41},"line",1,[38,43,44],{},"User-agent: *\n",[38,46,48],{"class":40,"line":47},2,[38,49,50],{},"Disallow: \u002Fadmin\u002F\n",[38,52,54],{"class":40,"line":53},3,[38,55,56],{},"Sitemap: https:\u002F\u002Fexample.com\u002Fsitemap.xml\n",[14,58,59,63,64,67,68,71,72,75,76,79],{},[60,61,62],"strong",{},"User-agent","：适用哪个爬虫，",[35,65,66],{},"*"," 表示所有。",[60,69,70],{},"Disallow","：禁止访问的路径。",[60,73,74],{},"Allow","：例外路径。",[60,77,78],{},"Sitemap","：Sitemap 文件位置。",[24,81,82],{"id":82},"应该屏蔽什么",[14,84,85],{},"后台管理页面、站内搜索结果、筛选\u002F分面 URL、测试环境、登录页面、API 端点。不要屏蔽影响页面渲染的 CSS、JS 或图片文件。",[24,87,88],{"id":88},"不应屏蔽什么",[14,90,91,92,95],{},"绝不要屏蔽你希望被索引的页面。用 ",[35,93,94],{},"noindex"," 元标签控制索引。Robots.txt 只控制抓取，不直接控制索引。",[24,97,98],{"id":98},"测试方法",[14,100,101,102,105],{},"使用我们的 ",[18,103,104],{"href":20},"Robots.txt 生成与测试工具","：粘贴 URL 查看是否会被规则拦截。同时使用 Google Search Console 的 robots.txt 测试器。",[24,107,108],{"id":108},"常见错误",[14,110,111,112,115],{},"屏蔽 CSS\u002FJS（破坏渲染）、使用 ",[35,113,114],{},"Disallow: \u002F"," 屏蔽整站、忘记添加 sitemap 位置、将 robots.txt 当作安全工具。",[24,117,118],{"id":118},"核心要点",[14,120,121,122,125],{},"屏蔽低价值路径，保持 CSS\u002FJS 可访问，用我们的 ",[18,123,124],{"href":20},"Robots.txt 测试器"," 验证规则，添加 sitemap 链接。",[127,128],"hr",{},[14,130,131],{},[132,133,134,135,138],"em",{},"使用",[18,136,137],{"href":20},"免费 Robots.txt 生成器","创建你的抓取规则。",[140,141,142],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":33,"searchDepth":47,"depth":47,"links":144},[145,146,147,148,149,150],{"id":26,"depth":47,"text":26},{"id":82,"depth":47,"text":82},{"id":88,"depth":47,"text":88},{"id":98,"depth":47,"text":98},{"id":108,"depth":47,"text":108},{"id":118,"depth":47,"text":118},"实用的 robots.txt 教程。涵盖语法规则、最佳实践、规则生成及 URL 拦截测试方法。","md",{"date":154,"category":155,"tags":156,"relatedTools":160,"image":162},"2026-03-20","technical-seo",[157,158,159],"robots.txt","抓取预算","技术SEO",[161],"robots-txt-generator","\u002Fsite-icon.png",true,"\u002Fblog\u002Fzh\u002Frobots-txt-guide",{"title":5,"description":151},"blog\u002Fzh\u002Frobots-txt-guide","ZdFLWEv15p9St1Gc1oOUSZT8pknCmjStuT8nsaxHakE",1780480145251]