#==============================================
# ROBOTS.TXT RECOMMENDED SETTINGS
#==============================================
User-agent: * 
Crawl-Delay: 30  #Keep a 30 second delay to help prevent bot spamming

# Block all bots by default
User-agent: *
Disallow: /

#----------------------------------------------
# Allow trusted search engine bots
#----------------------------------------------
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Baiduspider
Allow: /

#----------------------------------------------
# Block AI Bots
#----------------------------------------------
# Allow AI bots (optional - e.g., GPTBot)
# User-agent: GPTBot
# Allow: /
User-agent: GPTBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

#----------------------------------------------
# Protect sensitive directories
#----------------------------------------------
User-agent: *
Disallow: /wp-admin/             #block access to admin section
Disallow: /private/
Disallow: /config/
Disallow: /wp-login.php          #block access to admin section
Disallow: /search/               #block access to internal search result pages
Disallow: *?s=*                  #block access to internal search result pages
Disallow: *?p=*                  #block access to pages for which permalinks fails
Disallow: *&p=*                  #block access to pages for which permalinks fails
Disallow: *&preview=*            #block access to preview pages
Disallow: /tag/                  #block access to tag pages
Disallow: /author/               #block access to author pages
Disallow: /404-error/            #block access to 404 page

#----------------------------------------------
# Point to sitemap - change URL & uncomment
#----------------------------------------------
Sitemap: https://norbest.com/sitemap_index.xml
#==============================================