# Robots.txt for Housing Trust Silicon Valley
# Updated: September 2025

# Block specific unwanted bots
User-agent: Amazonbot/0.1
Disallow: /

User-agent: BLEXBot/1.0
Disallow: /

User-agent: Barkrowler/0.9
Disallow: /

User-agent: GPTBot/1.2
Disallow: /

# Block additional AI training bots
User-agent: ChatGPT-User
Disallow: /

User-agent: OpenAI-GPT
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: AI2Bot
Disallow: /

# Block other unwanted crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /events/list/?*eventDisplay=past
Disallow: /events/category/*/list/?*eventDisplay=past

# Allow major search engines
User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/*/inc/
Disallow: /trackback/
Disallow: /comments/
Disallow: /xmlrpc.php
Disallow: /*?
Disallow: /*?*
Allow: /wp-content/uploads/

User-agent: Bingbot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/*/inc/
Disallow: /trackback/
Disallow: /comments/
Disallow: /xmlrpc.php
Disallow: /*?
Disallow: /*?*
Allow: /wp-content/uploads/

# Rules for all other crawlers
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/*/inc/
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /comments/
Disallow: /search/
Disallow: /author/
Disallow: /users/
Disallow: /xmlrpc.php
Disallow: /*?
Disallow: /*?*
Disallow: /wp-*.php
Disallow: /readme.html
Disallow: /license.txt

# Allow important content
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*/css/
Allow: /wp-content/themes/*/js/
Allow: /wp-content/themes/*/images/
Allow: /*.css$
Allow: /*.js$

# Sitemap location
Sitemap: https://housingtrustsv.org/sitemap_index.xml
Sitemap: https://housingtrustsv.org/sitemap.xml

# Crawl delay for non-major search engines (reduce server load)
User-agent: *
Crawl-delay: 25