User-Agent: *
Disallow: 
Disallow: /xmlrpc.php
Disallow: /startup/
Disallow: /wp-admin/
Disallow: /superadmin/

# WP Import Export Rule
User-agent: *
Disallow: /wp-content/uploads/wp-import-export-lite/

# Allow: Search engine bots
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Yandex
Disallow:

# Allow: AI crawlers
User-agent: GPTBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: OAI-SearchBot
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: Claude-Web
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: Perplexity-User
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: Applebot
Disallow:

User-agent: Applebot-Extended
Disallow:

User-agent: Bytespider
Disallow:

User-agent: cohere-ai
Disallow:

# Block: Malicious / scraping bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Sitemap location
Sitemap: http://www.ukstartups.org/sitemap_index.xml