# Disallow all crawlers access to certain pages.

User-agent: *
Disallow: /asp/
Disallow: /php/
Disallow: /my_membership/member_processing.php
Disallow: /my_membership/join.php
Disallow: /my_membership/login.php
Disallow: /my_membership/edit.php
Disallow: /my_membership/lost.php
Disallow: /my_membership/submitted_orders.php
Disallow: /products/products.php?*As=on*
Disallow: /products/products.php?*Fd=region*

# Agent Specific Sections

User-agent: Fatbot
Disallow: /products/
User-agent: ShopWiki
Disallow: /products/
Crawl-Delay: 10
User-agent: twiceler
Crawl-Delay: 10
Disallow: /products/
Disallow: /asp/
Disallow: /php/
Disallow: /my_membership/member_processing.php
Disallow: /my_membership/join.php
Disallow: /my_membership/login.php
Disallow: /my_membership/edit.php
Disallow: /my_membership/lost.php
Disallow: /my_membership/submitted_orders.php
User-agent: Twiceler-0.9
Crawl-Delay: 10
Disallow: /products/
Disallow: /asp/
Disallow: /php/
Disallow: /my_membership/member_processing.php
Disallow: /my_membership/join.php
Disallow: /my_membership/login.php
Disallow: /my_membership/edit.php
Disallow: /my_membership/lost.php
Disallow: /my_membership/submitted_orders.php
User-agent: Webwombat
Crawl-Delay: 10
User-agent: archive.org_bot
Crawl-Delay: 30
#Disallow: /
User-agent: panscient.com
Disallow: /
User-agent: turnitinbot
Disallow: /
User-agent: Gaisbot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: ScoutJet
Allow: /default.php
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: MLBot
Disallow: /
User-agent: jyxobot
Disallow: /
User-agent: eurobot
Disallow: /
User-agent: Steeler
Disallow: /
User-agent: exabot
Disallow: /products/products.php
User-agent: 008
Disallow: /
User-agent: NextGenSearchBot
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: discobot
Disallow: /
User-agent: TwengaBot
Disallow: /
User-agent: spbot
Disallow: /
User-agent: Linguee
Disallow: /
User-agent: cityreview
Disallow: /
User-agent: nodestackbot
Disallow: /
User-agent: Accelobot
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: ccbot
Disallow: /