Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

More test cases

  • Loading branch information...
commit 30a36492584e5b467a39cb8573742ed4c7e8bdca 1 parent effb899
@michaelklishin authored
View
193 test/resources/robots/alexa.com.txt
@@ -0,0 +1,193 @@
+# The crawlers listed below are allowed on the Alexa site.
+# Alexa allows other crawlers on a case by case basis.
+
+#
+# Alexa provides access to traffic ranking data via Amazon Web Services.
+# More information here: <URL: http://aws.amazon.com/awis>
+
+User-agent: Mediapartners-Google*
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+User-agent: googlemediabot*
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+User-agent: googlebot
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+Disallow: /search
+Disallow: /data/
+Disallow: /site/linksin
+Disallow: /virtual
+Disallow: /edit
+Disallow: /reset
+Disallow: /profile
+Disallow: /write
+Disallow: /siteaudit/visibility
+Disallow: /login
+Disallow: /logout
+Disallow: /register
+Disallow: /accept
+Disallow: /audituser
+Disallow: /startreport
+Disallow: /quickregister
+Disallow: /account
+Disallow: /reviews/single
+
+User-agent: gulliver
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: slurp
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+User-agent: fast
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: scooter
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: vscooter
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: ia_archiver
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: Nutch
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: FAST-WebCrawler
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: teoma
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: Lycos_Spider_(T-Rex)
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: bingbot
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+User-agent: MSNBot
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+User-agent: MSNPTC
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: msnbot-media/1.0
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: ArchitextSpider
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: ZyBorg
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: SurveyBot
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: AlexaBot
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: Baiduspider
+Disallow: /cgi-bin
+Disallow: /search
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+
+User-agent: Yandex
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+User-agent: JikeSpider
+Disallow: /cgi-bin
+Disallow: /edit
+Disallow: /toolbar/success
+Disallow: /help/viewforum.php
+Disallow: /site/clickstream
+
+# Disallow all other crawlers
+
+User-agent: *
+Disallow: /
View
138 test/resources/robots/amazon.com.txt
@@ -0,0 +1,138 @@
+# Disallow all crawlers access to certain pages.
+
+User-agent: *
+Disallow: /exec/obidos/account-access-login
+Disallow: /exec/obidos/change-style
+Disallow: /exec/obidos/flex-sign-in
+Disallow: /exec/obidos/handle-buy-box
+Disallow: /exec/obidos/tg/cm/member/
+Disallow: /gp/cart
+Disallow: /gp/flex
+Disallow: /gp/product/e-mail-friend
+Disallow: /gp/product/product-availability
+Disallow: /gp/product/rate-this-item
+Disallow: /gp/sign-in
+Disallow: /gp/reader
+Disallow: /gp/sitbv3/reader
+Disallow: /gp/richpub/syltguides/create
+Disallow: /gp/gfix
+Disallow: /gp/associations/wizard.html
+Disallow: /gp/dmusic/order
+Disallow: /gp/legacy-handle-buy-box.html
+Disallow: /gp/aws/ssop
+Disallow: /gp/yourstore
+Disallow: /gp/gift-central/organizer/add-wishlist
+Disallow: /gp/vote
+Disallow: /gp/voting/
+Disallow: /gp/music/wma-pop-up
+Disallow: /gp/customer-images
+Disallow: /gp/richpub/listmania/createpipeline
+Disallow: /gp/content-form
+Disallow: /gp/pdp/invitation/invite
+Disallow: /gp/customer-reviews/common/du
+Disallow: /gp/customer-reviews/write-a-review.html
+Disallow: /gp/associations/wizard.html
+Disallow: /gp/music/clipserve
+Disallow: /gp/offer-listing
+Disallow: /gp/customer-media/upload
+Disallow: /gp/history
+Disallow: /gp/item-dispatch
+Disallow: /gp/dmusic/order/handle-buy-box.html
+Disallow: /gp/recsradio
+Disallow: /gp/slredirect
+Disallow: /dp/shipping/
+Disallow: /dp/twister-update/
+Disallow: /dp/manual-submit/
+Disallow: /dp/e-mail-friend/
+Disallow: /dp/product-availability/
+Disallow: /dp/rate-this-item/
+Disallow: /gp/registry/wishlist/*/reserve
+Disallow: /gp/structured-ratings/actions/get-experience.html
+Disallow: /gp/twitter/
+Disallow: /ap/signin
+Disallow: /gp/registry/wishlist/*
+Disallow: /wishlist/*
+Allow: /wishlist/universal*
+Allow: /wishlist/vendor-button*
+Allow: /wishlist/get-button*
+Disallow: /gp/wishlist/*
+Allow: /gp/wishlist/universal*
+Allow: /gp/wishlist/vendor-button*
+Allow: /gp/wishlist/ipad-install*
+Disallow: /registry/wishlist/*
+Disallow: /review/common/du
+
+User-Agent: Googlebot
+Disallow: /rss/people/*/reviews
+Disallow: /gp/pdp/rss/*/reviews
+Disallow: /gp/cdp/member-reviews/
+Disallow: /gp/aw/cr/
+Disallow: /exec/obidos/account-access-login
+Disallow: /exec/obidos/change-style
+Disallow: /exec/obidos/flex-sign-in
+Disallow: /exec/obidos/handle-buy-box
+Disallow: /exec/obidos/tg/cm/member/
+Disallow: /gp/cart
+Disallow: /gp/flex
+Disallow: /gp/product/e-mail-friend
+Disallow: /gp/product/product-availability
+Disallow: /gp/product/rate-this-item
+Disallow: /gp/sign-in
+Disallow: /gp/reader
+Disallow: /gp/sitbv3/reader
+Disallow: /gp/richpub/syltguides/create
+Disallow: /gp/gfix
+Disallow: /gp/associations/wizard.html
+Disallow: /gp/dmusic/order
+Disallow: /gp/legacy-handle-buy-box.html
+Disallow: /gp/aws/ssop
+Disallow: /gp/yourstore
+Disallow: /gp/gift-central/organizer/add-wishlist
+Disallow: /gp/vote
+Disallow: /gp/voting/
+Disallow: /gp/music/wma-pop-up
+Disallow: /gp/customer-images
+Disallow: /gp/richpub/listmania/createpipeline
+Disallow: /gp/content-form
+Disallow: /gp/pdp/invitation/invite
+Disallow: /gp/customer-reviews/common/du
+Disallow: /gp/customer-reviews/write-a-review.html
+Disallow: /gp/associations/wizard.html
+Disallow: /gp/music/clipserve
+Disallow: /gp/offer-listing
+Disallow: /gp/customer-media/upload
+Disallow: /gp/history
+Disallow: /gp/item-dispatch
+Disallow: /gp/dmusic/order/handle-buy-box.html
+Disallow: /gp/recsradio
+Disallow: /gp/slredirect
+Disallow: /dp/shipping/
+Disallow: /dp/twister-update/
+Disallow: /dp/manual-submit/
+Disallow: /dp/e-mail-friend/
+Disallow: /dp/product-availability/
+Disallow: /dp/rate-this-item/
+Disallow: /gp/registry/wishlist/*/reserve
+Disallow: /gp/structured-ratings/actions/get-experience.html
+Disallow: /gp/twitter/
+Disallow: /ap/signin
+Disallow: /gp/registry/wishlist/*
+Disallow: /wishlist/*
+Allow: /wishlist/universal*
+Allow: /wishlist/vendor-button*
+Allow: /wishlist/get-button*
+Disallow: /gp/wishlist/*
+Allow: /gp/wishlist/universal*
+Allow: /gp/wishlist/vendor-button*
+Allow: /gp/wishlist/ipad-install*
+Disallow: /registry/wishlist/*
+Disallow: /review/common/du
+
+# Sitemap files
+Sitemap: http://www.amazon.com/sitemap-manual-index.xml
+Sitemap: http://www.amazon.com/sitemap_dp_index.xml
+Sitemap: http://www.amazon.com/sitemap_vendor_videos_us.xml
+Sitemap: http://www.amazon.com/sitemap_vod_index.xml
+Sitemap: http://www.amazon.com/sitemap_entity_index.xml.gz
+Sitemap: http://www.amazon.com/sitemaps.US_detail_page_sitemap_mobile_index.xml.gz
+Sitemap: http://www.amazon.com/sitemaps.US_detail_page_sitemap_desktop_index.xml.gz
View
3  test/resources/robots/qq.com.txt
@@ -0,0 +1,3 @@
+User-agent: *
+Disallow:
+Sitemap: http://www.qq.com/sitemap_index.xml
Please sign in to comment.
Something went wrong with that request. Please try again.