# Disallow all crawlers access to certain pages.

User-agent: *
Disallow: /asp
Disallow: /php
Disallow: /my_membership/member_processing.php
Disallow: /my_membership/join.php
Disallow: /my_membership/login.php
Disallow: /my_membership/edit.php
Disallow: /my_membership/lost.php
Disallow: /my_membership/submitted_orders.php

# Agent Specific Sections

User-agent: Fatbot
Disallow: /products
User-agent: ShopWiki
Disallow: /products
Crawl-Delay: 10
User-agent: Twiceler
Disallow: /
User-agent: Twiceler-0.9
Disallow: /
User-agent: Webwombat
Crawl-Delay: 10
User-agent: archive.org_bot
#Crawl-Delay: 30
Disallow: /
User-agent: panscient.com
Disallow: /
User-agent: turnitinbot
Disallow: /
User-agent: Gaisbot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: ScoutJet
Allow: /default.php
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: MLBot
Disallow: /
User-agent: jyxobot
Disallow: /
User-agent: eurobot
Disallow: /
User-agent: Steeler
Disallow: /
User-agent: exabot
Disallow: /products/products.php
User-agent: 008
Disallow: /
User-agent: NextGenSearchBot
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: discobot
Disallow: /