[{"data":1,"prerenderedAt":130},["ShallowReactive",2],{"blog-post-robots-txt-guide-en":3},{"id":4,"title":5,"body":6,"description":114,"extension":115,"meta":116,"navigation":125,"path":126,"seo":127,"stem":128,"__hash__":129},"blog\u002Fblog\u002Fen\u002Frobots-txt-guide.md","Robots.txt Guide for SEO",{"type":7,"value":8,"toc":107},"minimark",[9,13,23,28,58,62,65,69,76,80,83,87,90,93,103],[10,11,5],"h1",{"id":12},"robotstxt-guide-for-seo",[14,15,16,17,22],"p",{},"Robots.txt tells crawlers which pages to visit and which to skip. A good file protects crawl budget. Use our ",[18,19,21],"a",{"href":20},"\u002Ftools\u002Frobots-txt-generator","Robots.txt Generator"," to create one.",[24,25,27],"h2",{"id":26},"basic-syntax","Basic Syntax",[29,30,35],"pre",{"className":31,"code":32,"language":33,"meta":34,"style":34},"language-txt shiki shiki-themes github-light github-dark","User-agent: *\nDisallow: \u002Fadmin\u002F\nSitemap: https:\u002F\u002Fexample.com\u002Fsitemap.xml\n","txt","",[36,37,38,46,52],"code",{"__ignoreMap":34},[39,40,43],"span",{"class":41,"line":42},"line",1,[39,44,45],{},"User-agent: *\n",[39,47,49],{"class":41,"line":48},2,[39,50,51],{},"Disallow: \u002Fadmin\u002F\n",[39,53,55],{"class":41,"line":54},3,[39,56,57],{},"Sitemap: https:\u002F\u002Fexample.com\u002Fsitemap.xml\n",[24,59,61],{"id":60},"what-to-block","What to Block",[14,63,64],{},"Admin panels, search results, staging sites. Never block CSS or JS files.",[24,66,68],{"id":67},"testing","Testing",[14,70,71,72,75],{},"Use our ",[18,73,74],{"href":20},"Robots.txt Tester"," or Google Search Console.",[24,77,79],{"id":78},"common-mistakes","Common Mistakes",[14,81,82],{},"Blocking CSS\u002FJS, using Disallow: \u002F for everything, forgetting sitemap location.",[24,84,86],{"id":85},"key-takeaways","Key Takeaways",[14,88,89],{},"Block low-value paths, test rules, add sitemap. Robots.txt is not a security tool.",[91,92],"hr",{},[14,94,95],{},[96,97,98,99,102],"em",{},"Generate your robots.txt with our ",[18,100,101],{"href":20},"free tool",".",[104,105,106],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":34,"searchDepth":48,"depth":48,"links":108},[109,110,111,112,113],{"id":26,"depth":48,"text":27},{"id":60,"depth":48,"text":61},{"id":67,"depth":48,"text":68},{"id":78,"depth":48,"text":79},{"id":85,"depth":48,"text":86},"Learn robots.txt syntax, best practices, and how to test URL blocking with our free generator.","md",{"date":117,"category":118,"tags":119,"relatedTools":122,"image":124},"2026-03-20","technical-seo",[120,121],"robots.txt","technical seo",[123],"robots-txt-generator","\u002Fsite-icon.png",true,"\u002Fblog\u002Fen\u002Frobots-txt-guide",{"title":5,"description":114},"blog\u002Fen\u002Frobots-txt-guide","NaY1geMdAHgW-FvqMGzr93ZiuA1LzYZMjJcrAdLan20",1780480145242]