User-agent: *
Disallow: /r/candidates/
Disallow: /c/companies/
Disallow: /
Disallow: /btrp/
User-agent: bingbot
Crawl-delay: 1
Disallow: /r/candidates/
Disallow: /c/companies/
User-agent: gsa-crawler
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: Mediapartners-Google
Allow: /
User-agent: Googlebot
Disallow: /r/candidates/
Disallow: /c/companies/
User-agent: Slurp
Crawl-delay: 1
Disallow: /r/candidates/
Disallow: /c/companies/