#Basic /robots.txt file
#User-agent: webcrawler
#Disallow:
#User-agent: lycra
#Disallow: /

#Force Google to stop trying to retrieve this old file
User-agent: FeedFetcher-Google-CoOp
Disallow: /google_cse5.php

User-agent: *
Disallow: /xhr/
Disallow: /api/
Disallow: /redis/
Disallow: /dialog/
Disallow: /widget/
Disallow: /facebook/
Disallow: /google.com/