# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism User-agent: webcrawler Disallow: User-agent: lycra Disallow: User-agent: * Disallow: /Eventsdata/* User-agent: * Disallow: /VideoLibrary/* User-agent: * Disallow: /venuedetails.aspx User-agent: * Disallow: /register.aspx User-agent: * Disallow: /enquiry.aspx User-agent: * Disallow: /tellafriend.aspx #User-agent: * #Sitemap: www.cmtevents.com/sitemap.aspx crawl-delay:10