# Block AI crawlers and scrapers
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: CCBot
Disallow: /

# Allow specific directories with delay
User-agent: *
Crawl-delay: 10

# Block sensitive areas
Disallow: /ghost/
Disallow: /p/
Disallow: /signup/
Disallow: /signin/
Disallow: /account/
Disallow: /members/
Disallow: /private/
Disallow: /admin/
Disallow: /search/
Disallow: /preview/
Disallow: /?s=*

# Allow content areas
Allow: /tag/
Allow: /page/
Allow: /author/
Allow: /post/
Allow: /content/images/
Allow: /sitemap.xml
Allow: /rss/

# Image indexing
User-agent: Googlebot-Image
Allow: /content/images/

# News indexing
User-agent: Googlebot-News
Allow: /

# XML Sitemaps
Sitemap: https://foleto.com/sitemap.xml
Sitemap: https://foleto.com/sitemap-posts.xml
Sitemap: https://foleto.com/sitemap-pages.xml
Sitemap: https://foleto.com/sitemap-authors.xml
Sitemap: https://foleto.com/sitemap-tags.xml