#deny 80legs.com crawler
User-agent: 008
Disallow: /

User-agent: simplepie
Disallow: /

User-agent: 80legs.com
Disallow: /

User-agent: curl
Disallow: /

User-agent: Python urllib
Disallow: /

User-agent: OSCE
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Wotbox
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Genieo
Disallow: /

User-agent: JobdiggerSpider
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Abonti
Disallow: /

User-agent: LinkChecker
Disallow: /

User-agent: Jetslide
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: SISTRIX
Disallow: /

User-agent: WBSearchBot
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: eknip
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: KINGSpider
Disallow: /

User-agent: Twitterbot
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: *
Disallow: ?
Disallow: /aspx/
Allow: /aspx/read.aspx
Disallow: /*?*pdf=true*
Disallow: /*?*PDF=true*
Disallow: PDF=true
Disallow: *PDF=true*
Disallow: *export=pdf*
Disallow: export=pdf
Disallow: /*?*rss=true*
Disallow: /*?*CalDtm=*
Disallow: /*?*zoeken_term=*
Disallow: /*?*Zoe=*
Disallow: /*?*zoeken_metwildcard=true*
Disallow: /*?*pager_page*
crawl-delay: 30