# robots.txt for https://payangel.com
# Last updated: 2026-07-28

# ---------------------------------------------------------------
# Default rules — all crawlers
# ---------------------------------------------------------------
User-agent: *
Allow: /

# WordPress admin & system paths
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /*?replytocom=

# Internal search results (crawl waste, thin pages)
Disallow: /?s=
Disallow: /search/

# Feeds (duplicate content of posts/pages)
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/

# Faceted / tracking / session parameters
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?msclkid=
Disallow: /*?ref=
Disallow: /*?orderby=
Disallow: /*?filter_
Disallow: /*?add-to-cart=
Disallow: /*?p=

# Transactional / account pages (no SEO value, may contain user data)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /thank-you/
Disallow: /order-received/

# Never block rendering assets — Google must fetch CSS, JS and images
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.gif$

# ---------------------------------------------------------------
# AI / answer-engine crawlers — explicitly allowed for AEO visibility
# (PayAngel wants to be cited in AI answers about UK/US → Africa remittance)
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: MistralAI-User
Allow: /

# ---------------------------------------------------------------
# Aggressive / low-value scrapers — throttled or blocked
# ---------------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

# ---------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------
Sitemap: https://payangel.com/sitemap_index.xml