# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism #User-agent:* #Disallow:/ User-agent:* Allow:/main.aspx Allow:/aboutevent.aspx Allow:/eventschdule.aspx Allow:/speakerprofile.aspx User-agent: lycra Disallow:/ User-agent:* Disallow: /eventdatas/ Disallow: /eventdatas User-agent:* Disallow: /album/ User-agent:* Disallow: /newevents/ User-agent:* Disallow: /VideoLibrary/ User-agent:* Disallow: /final/ User-agent:* Disallow: /venuedetails.aspx User-agent:* Disallow: /printerfriendly.aspx User-agent:* Disallow: /register.aspx User-agent:* Disallow: /mediadownload.aspx User-agent:* Disallow: /paperproposal.aspx User-agent:* Disallow: /eventbrochure.aspx User-agent:* Disallow: /singnup.aspx User-agent:* Disallow: /enquiry.aspx User-agent:* Disallow: /tellafriend.aspx #User-agent:* #Sitemap: www.cmtevents.com/sitemapxml.aspx #crawl-delay:10