# For use by search.w3.org
User-agent: W3C-gsa
Disallow: 

# W3C Link checker
User-agent: W3C-checklink
Disallow:

#Rambler
User-agent: StackRambler
Disallow:

#Google
User-agent: Googlebot
Disallow:

#Google Mediapartners-Google
User-agent: Mediapartners-Google
Disallow:

#Google Adsbot-Google
User-agent: Adsbot-Google
Disallow:

#Google Googlebot-Image
User-agent: Googlebot-Image
Disallow:

#robots.txt for http://xbt.dp.ua
User-agent: *

Disallow: /1/
Disallow: /admin/
Disallow: /blocks/
Disallow: /cashe/
Disallow: /cache/
Disallow: /xcache/
Disallow: /download/
Disallow: /editor/
Disallow: /errors/
#Disallow: /forum/
Disallow: /image/
Disallow: /images/
Disallow: /include/
Disallow: /js/
Disallow: /languages/
Disallow: /net/
Disallow: /phpadmin1/
Disallow: /pic/
Disallow: /themes/
Disallow: /torrentbar/
Disallow: /torrents/

Sitemap: http://www.xbt.dp.ua/sitemap.xml