From 2ac4fc630a1ed0b50b8716c420f7d86c12166709 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 4 Jan 2016 22:15:45 +0900 Subject: [PATCH] Update real world tests, introducing Webmock+VCR --- test/helper.rb | 7 + test/test_webrobots.rb | 38 +- test/vcr_cassettes/major_sites.yml | 2006 ++++++++++++++++++++++++++++ webrobots.gemspec | 3 + 4 files changed, 2035 insertions(+), 19 deletions(-) create mode 100644 test/vcr_cassettes/major_sites.yml diff --git a/test/helper.rb b/test/helper.rb index 4bde52b..b499f52 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -9,6 +9,13 @@ end require 'test/unit' require 'shoulda' +require 'webmock/test_unit' +require 'vcr' + +VCR.configure do |c| + c.cassette_library_dir = 'test/vcr_cassettes' + c.hook_into :webmock +end $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) $LOAD_PATH.unshift(File.dirname(__FILE__)) diff --git a/test/test_webrobots.rb b/test/test_webrobots.rb index af6d605..960d285 100644 --- a/test/test_webrobots.rb +++ b/test/test_webrobots.rb @@ -531,28 +531,28 @@ class TestWebRobots < Test::Unit::TestCase context "robots.txt in the real world" do setup do @testbot = WebRobots.new('TestBot') - @msnbot = WebRobots.new('TestMSNBot') # matches msnbot + @baidubot = WebRobots.new('TestBaiduspider') # matches Baiduspider end should "be parsed for major sites" do - assert_nothing_raised { - assert !@testbot.allowed?("http://www.google.com/search") - assert !@testbot.allowed?("https://www.google.com/search") - assert !@testbot.allowed?("http://www.google.com/news/section?pz=1&cf=all&ned=jp&topic=y&ict=ln") - assert @testbot.allowed?("http://www.google.com/news/directory?pz=1&cf=all&ned=us&hl=en&sort=users&category=6") - } - assert_nothing_raised { - assert @testbot.allowed?("http://www.yahoo.com/") - assert !@testbot.allowed?("http://www.yahoo.com/?") - assert !@testbot.allowed?("http://www.yahoo.com/p/foo") - } - assert_nothing_raised { - assert !@testbot.allowed?("http://store.apple.com/vieworder") - assert @msnbot.allowed?("http://store.apple.com/vieworder") - } - assert_nothing_raised { - assert !@testbot.allowed?("http://github.com/login") - } + VCR.use_cassette("major_sites") do + assert_nothing_raised { + assert !@testbot.allowed?("https://www.google.com/search") + assert !@testbot.allowed?("https://www.google.com/catalogs") + assert @testbot.allowed?("https://www.google.com/catalogs/about") + } + assert_nothing_raised { + assert @testbot.allowed?("https://www.yahoo.com/") + assert !@testbot.allowed?("https://www.yahoo.com/p/foo") + } + assert_nothing_raised { + assert @testbot.allowed?("http://www.apple.com/jp/foo") + assert !@baidubot.allowed?("http://www.apple.com/jp/foo") + } + assert_nothing_raised { + assert !@testbot.allowed?("https://github.com/login") + } + end end end diff --git a/test/vcr_cassettes/major_sites.yml b/test/vcr_cassettes/major_sites.yml new file mode 100644 index 0000000..1dc3e5a --- /dev/null +++ b/test/vcr_cassettes/major_sites.yml @@ -0,0 +1,2006 @@ +--- +http_interactions: +- request: + method: get + uri: https://www.google.com/robots.txt + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - TestBot + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Vary: + - Accept-Encoding + Content-Type: + - text/plain + Date: + - Mon, 04 Jan 2016 13:39:15 GMT + Expires: + - Mon, 04 Jan 2016 13:39:15 GMT + Cache-Control: + - private, max-age=0 + Last-Modified: + - Thu, 17 Dec 2015 01:06:42 GMT + X-Content-Type-Options: + - nosniff + Server: + - sffe + X-Xss-Protection: + - 1; mode=block + Alternate-Protocol: + - 443:quic,p=1 + Alt-Svc: + - quic="www.google.com:443"; ma=600; v="30,29,28,27,26,25",quic=":443"; ma=600; + v="30,29,28,27,26,25" + Transfer-Encoding: + - chunked + body: + encoding: ASCII-8BIT + string: |+ + User-agent: * + Disallow: /search + Allow: /search/about + Disallow: /sdch + Disallow: /groups + Disallow: /catalogs + Allow: /catalogs/about + Allow: /catalogs/p? + Disallow: /catalogues + Allow: /newsalerts + Disallow: /news + Allow: /news/directory + Disallow: /nwshp + Disallow: /setnewsprefs? + Disallow: /index.html? + Disallow: /? + Allow: /?hl= + Disallow: /?hl=*& + Allow: /?hl=*&gws_rd=ssl$ + Disallow: /?hl=*&*&gws_rd=ssl + Allow: /?gws_rd=ssl$ + Allow: /?pt1=true$ + Disallow: /addurl/image? + Allow: /mail/help/ + Disallow: /mail/ + Disallow: /pagead/ + Disallow: /relpage/ + Disallow: /relcontent + Disallow: /imgres + Disallow: /imglanding + Disallow: /sbd + Disallow: /keyword/ + Disallow: /u/ + Disallow: /univ/ + Disallow: /cobrand + Disallow: /custom + Disallow: /advanced_group_search + Disallow: /googlesite + Disallow: /preferences + Disallow: /setprefs + Disallow: /swr + Disallow: /url + Disallow: /default + Disallow: /m? + Disallow: /m/ + Allow: /m/finance + Disallow: /wml? + Disallow: /wml/? + Disallow: /wml/search? + Disallow: /xhtml? + Disallow: /xhtml/? + Disallow: /xhtml/search? + Disallow: /xml? + Disallow: /imode? + Disallow: /imode/? + Disallow: /imode/search? + Disallow: /jsky? + Disallow: /jsky/? + Disallow: /jsky/search? + Disallow: /pda? + Disallow: /pda/? + Disallow: /pda/search? + Disallow: /sprint_xhtml + Disallow: /sprint_wml + Disallow: /pqa + Disallow: /palm + Disallow: /gwt/ + Disallow: /purchases + Disallow: /bsd? + Disallow: /linux? + Disallow: /mac? + Disallow: /microsoft? + Disallow: /unclesam? + Disallow: /answers/search?q= + Disallow: /local? + Disallow: /local_url + Disallow: /shihui? + Disallow: /shihui/ + Disallow: /froogle? + Disallow: /products? + Disallow: /froogle_ + Disallow: /product_ + Disallow: /products_ + Disallow: /products; + Disallow: /print + Disallow: /books/ + Disallow: /bkshp?*q=* + Disallow: /books?*q=* + Disallow: /books?*output=* + Disallow: /books?*pg=* + Disallow: /books?*jtp=* + Disallow: /books?*jscmd=* + Disallow: /books?*buy=* + Disallow: /books?*zoom=* + Allow: /books?*q=related:* + Allow: /books?*q=editions:* + Allow: /books?*q=subject:* + Allow: /books/about + Allow: /booksrightsholders + Allow: /books?*zoom=1* + Allow: /books?*zoom=5* + Disallow: /ebooks/ + Disallow: /ebooks?*q=* + Disallow: /ebooks?*output=* + Disallow: /ebooks?*pg=* + Disallow: /ebooks?*jscmd=* + Disallow: /ebooks?*buy=* + Disallow: /ebooks?*zoom=* + Allow: /ebooks?*q=related:* + Allow: /ebooks?*q=editions:* + Allow: /ebooks?*q=subject:* + Allow: /ebooks?*zoom=1* + Allow: /ebooks?*zoom=5* + Disallow: /patents? + Disallow: /patents/download/ + Disallow: /patents/pdf/ + Disallow: /patents/related/ + Disallow: /scholar + Disallow: /citations? + Allow: /citations?user= + Disallow: /citations?*cstart= + Allow: /citations?view_op=new_profile + Allow: /citations?view_op=top_venues + Disallow: /complete + Disallow: /s? + Disallow: /sponsoredlinks + Disallow: /videosearch? + Disallow: /videopreview? + Disallow: /videoprograminfo? + Allow: /maps?*output=classic* + Allow: /maps?*file= + Allow: /maps/api/js? + Allow: /maps/d/ + Disallow: /maps? + Disallow: /mapstt? + Disallow: /mapslt? + Disallow: /maps/stk/ + Disallow: /maps/br? + Disallow: /mapabcpoi? + Disallow: /maphp? + Disallow: /mapprint? + Disallow: /maps/api/js/ + Disallow: /maps/api/staticmap? + Disallow: /maps/api/streetview + Disallow: /mld? + Disallow: /staticmap? + Disallow: /places/ + Allow: /places/$ + Disallow: /maps/preview + Disallow: /maps/place + Disallow: /help/maps/streetview/partners/welcome/ + Disallow: /help/maps/indoormaps/partners/ + Disallow: /lochp? + Disallow: /center + Disallow: /ie? + Disallow: /sms/demo? + Disallow: /katrina? + Disallow: /blogsearch? + Disallow: /blogsearch/ + Disallow: /blogsearch_feeds + Disallow: /advanced_blog_search + Disallow: /uds/ + Disallow: /chart? + Disallow: /transit? + Disallow: /mbd? + Disallow: /extern_js/ + Disallow: /xjs/ + Disallow: /calendar/feeds/ + Disallow: /calendar/ical/ + Disallow: /cl2/feeds/ + Disallow: /cl2/ical/ + Disallow: /coop/directory + Disallow: /coop/manage + Disallow: /trends? + Disallow: /trends/music? + Disallow: /trends/hottrends? + Disallow: /trends/viz? + Disallow: /trends/embed.js? + Disallow: /trends/fetchComponent? + Disallow: /notebook/search? + Disallow: /musica + Disallow: /musicad + Disallow: /musicas + Disallow: /musicl + Disallow: /musics + Disallow: /musicsearch + Disallow: /musicsp + Disallow: /musiclp + Disallow: /browsersync + Disallow: /call + Disallow: /archivesearch? + Disallow: /archivesearch/url + Disallow: /archivesearch/advanced_search + Disallow: /base/reportbadoffer + Disallow: /urchin_test/ + Disallow: /movies? + Disallow: /codesearch? + Disallow: /codesearch/feeds/search? + Disallow: /wapsearch? + Disallow: /safebrowsing + Allow: /safebrowsing/diagnostic + Allow: /safebrowsing/report_badware/ + Allow: /safebrowsing/report_error/ + Allow: /safebrowsing/report_phish/ + Disallow: /reviews/search? + Disallow: /orkut/albums + Allow: /jsapi + Disallow: /views? + Disallow: /c/ + Disallow: /cbk + Allow: /cbk?output=tile&cb_client=maps_sv + Disallow: /kh + Disallow: /vt + Disallow: /maps/vt + Disallow: /recharge/dashboard/car + Disallow: /recharge/dashboard/static/ + Disallow: /translate_a/ + Disallow: /translate_c + Disallow: /translate_f + Disallow: /translate_static/ + Disallow: /translate_suggestion + Disallow: /profiles/me + Allow: /profiles + Disallow: /s2/profiles/me + Allow: /s2/profiles + Allow: /s2/oz + Allow: /s2/photos + Allow: /s2/search/social + Allow: /s2/static + Disallow: /s2 + Disallow: /transconsole/portal/ + Disallow: /gcc/ + Disallow: /aclk + Disallow: /cse? + Disallow: /cse/home + Disallow: /cse/panel + Disallow: /cse/manage + Disallow: /tbproxy/ + Disallow: /imesync/ + Disallow: /shenghuo/search? + Disallow: /support/forum/search? + Disallow: /reviews/polls/ + Disallow: /hosted/images/ + Disallow: /ppob/? + Disallow: /ppob? + Disallow: /adwordsresellers + Disallow: /accounts/ClientLogin + Disallow: /accounts/ClientAuth + Disallow: /accounts/o8 + Allow: /accounts/o8/id + Disallow: /topicsearch?q= + Disallow: /xfx7/ + Disallow: /squared/api + Disallow: /squared/search + Disallow: /squared/table + Disallow: /toolkit/ + Allow: /toolkit/*.html + Disallow: /globalmarketfinder/ + Allow: /globalmarketfinder/*.html + Disallow: /qnasearch? + Disallow: /app/updates + Disallow: /sidewiki/entry/ + Disallow: /quality_form? + Disallow: /labs/popgadget/search + Disallow: /buzz/post + Disallow: /compressiontest/ + Disallow: /analytics/reporting/ + Disallow: /analytics/admin/ + Disallow: /analytics/web/ + Disallow: /analytics/feeds/ + Disallow: /analytics/settings/ + Allow: /alerts/manage + Allow: /alerts/remove + Disallow: /alerts/ + Allow: /alerts/$ + Disallow: /ads/search? + Disallow: /ads/plan/action_plan? + Disallow: /ads/plan/api/ + Disallow: /ads/hotels/partners + Disallow: /phone/compare/? + Disallow: /travel/clk + Disallow: /hotelfinder/rpc + Disallow: /hotels/rpc + Disallow: /flights/rpc + Disallow: /commercesearch/services/ + Disallow: /evaluation/ + Disallow: /chrome/browser/mobile/tour + Disallow: /compare/*/apply* + Disallow: /forms/perks/ + Disallow: /baraza/*/search + Disallow: /baraza/*/report + Disallow: /shopping/suppliers/search + Disallow: /ct/ + Disallow: /edu/cs4hs/ + Disallow: /trustedstores/s/ + Disallow: /trustedstores/tm2 + Disallow: /trustedstores/verify + Disallow: /adwords/proposal + Disallow: /shopping/product/ + Disallow: /shopping/seller + Disallow: /shopping/reviewer + Disallow: /about/careers/apply/ + Disallow: /about/careers/applications/ + Disallow: /landing/signout.html + Disallow: /webmasters/sitemaps/ping? + Disallow: /ping? + Allow: /gb/images + Allow: /gb/js + Disallow: /gallery/ + Allow: /chromecast/setup$ + Allow: /chromecast/setup/$ + Disallow: /landing/now/ontap/ + Sitemap: http://www.gstatic.com/culturalinstitute/sitemaps/www_google_com_culturalinstitute/sitemap-index.xml + Sitemap: http://www.gstatic.com/dictionary/static/sitemaps/sitemap_index.xml + Sitemap: http://www.gstatic.com/earth/gallery/sitemaps/sitemap.xml + Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml + Sitemap: http://www.gstatic.com/trends/websites/sitemaps/sitemapindex.xml + Sitemap: https://www.google.com/sitemap.xml + + http_version: + recorded_at: Mon, 04 Jan 2016 13:39:15 GMT +- request: + method: get + uri: https://www.yahoo.com/robots.txt + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - TestBot + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 04 Jan 2016 13:39:16 GMT + P3p: + - policyref="http://info.yahoo.com/w3c/p3p.xml", CP="CAO DSP COR CUR ADM DEV + TAI PSA PSD IVAi IVDi CONi TELo OTPi OUR DELi SAMi OTRi UNRi PUBi IND PHY + ONL UNI PUR FIN COM NAV INT DEM CNT STA POL HEA PRE LOC GOV" + Cache-Control: + - private + Last-Modified: + - Thu, 19 Nov 2015 00:31:02 GMT + Accept-Ranges: + - bytes + Vary: + - Accept-Encoding + Content-Type: + - text/plain; charset=utf-8 + Age: + - '2' + Via: + - http/1.1 usproxy2.fp.gq1.yahoo.com (ApacheTrafficServer), http/1.1 ir2.fp.sg3.yahoo.com + (ApacheTrafficServer) + Server: + - ATS + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Y-Trace: + - BAEAQAAAAAADb4_wm_pNEAAAAAAAAAAAGXeCK6w0SUwAAAAAAAAAAAAFKII_Uu55AAUogj9WSEhTfIBlAAAAAA-- + body: + encoding: ASCII-8BIT + string: | + User-agent: * + Disallow: /p/ + Disallow: /r/ + Disallow: /bin/ + Disallow: /includes/ + Disallow: /blank.html + Disallow: /_td_api + Disallow: /_tdpp_api + Disallow: /_remote + Disallow: /_multiremote + Disallow: /_tdhl_api + + Sitemap: https://www.yahoo.com/food/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/tech/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/travel/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/movies/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/beauty/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/health/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/style/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/makers/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/parenting/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/music/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/tv/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/politics/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/autos/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/katiecouric/sitemaps/sitemap_index_us_en-US.xml.gz + Sitemap: https://www.yahoo.com/digest/sitemap.xml + http_version: + recorded_at: Mon, 04 Jan 2016 13:39:17 GMT +- request: + method: get + uri: http://www.apple.com/robots.txt + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - TestBot + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Last-Modified: + - Thu, 22 Oct 2015 00:20:47 GMT + Server: + - Apache + Nncoection: + - close + Accept-Ranges: + - bytes + Content-Length: + - '6902' + Content-Type: + - text/plain; charset=UTF-8 + Cache-Control: + - max-age=561 + Expires: + - Mon, 04 Jan 2016 13:48:38 GMT + Date: + - Mon, 04 Jan 2016 13:39:17 GMT + Connection: + - keep-alive + body: + encoding: UTF-8 + string: "# robots.txt for http://www.apple.com/\n\nUser-agent: *\nDisallow: + /*/includes/*\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow: + /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/signed_in_account*\nDisallow: + /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/*WebObjects/*\nDisallow: + /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/answer/vote*\nDisallow: /*/shop/bag*\nDisallow: + /*/shop/browse/campaigns/mobile_overlay*\nDisallow: /*/shop/button_availability*\nDisallow: + /*/shop/favorites*\nDisallow: /*/shop/mobile/olss_error*\nDisallow: /*/shop/mobilex/*\nDisallow: + /*/shop/order/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow: + /*/shop/question/unsubscribe/\nDisallow: /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: + /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: /*/shop/socialsharing/*\nDisallow: + /*/shop/store/feeds/*\nDisallow: /*/shop/variationSelection\nDisallow: /*_adc_*/shop/\nDisallow: + /*_aoc_*/shop/\nDisallow: /*_edu_*/shop/\nDisallow: /*_enterprise*/shop/\nDisallow: + /*_internal-epp-discounted*/shop/\nDisallow: /*_k12nonbts*/shop/\nDisallow: + /*_kiosk*/shop/\nDisallow: /*_nonbts*/shop/\nDisallow: /*_qpromo*/shop/\nDisallow: + /*_refurb-discounted*/shop/\nDisallow: /cn/se/*\nDisallow: /retail/availability*\nDisallow: + /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow: + /search/*\nDisallow: /shop/*signed_in_account*\nDisallow: /shop/*sign_in*\nDisallow: + /shop/*sign_out*\nDisallow: /shop/*WebObjects/*\nDisallow: /shop/1-800-MY-APPLE/*\nDisallow: + /shop/answer/vote*\nDisallow: /shop/bag*\nDisallow: /shop/browse/campaigns/mobile_overlay*\nDisallow: + /shop/button_availability*\nDisallow: /shop/favorites*\nDisallow: /shop/mobile/olss_error*\nDisallow: + /shop/mobilex/\nDisallow: /shop/order/*\nDisallow: /shop/question/answer/report*\nDisallow: + /shop/question/subscribe*\nDisallow: /shop/question/unsubscribe/\nDisallow: + /shop/review/vote*\nDisallow: /shop/reviews/report*\nDisallow: /shop/rs-mvt/rel/*\nDisallow: + /shop/search/*\nDisallow: /shop/sentry*\nDisallow: /shop/socialsharing/*\nDisallow: + /shop/store/feeds/*\nDisallow: /shop/variationSelection*\nDisallow: /tmall*\nAllow: + /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent: + Baiduspider\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow: + /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow: + /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow: + /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow: + /*/shop/browse/overlay*\nDisallow: /*/shop/button_availability*\nDisallow: + /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: /*/shop/checkout*\nDisallow: + /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: /*/shop/identify_user*\nDisallow: + /*/shop/mobile/checkout/start*\nDisallow: /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: + /*/shop/question/subscribe*\nDisallow: /*/shop/question/unsubscribe/*\nDisallow: + /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: + /*/shop/sentry*\nDisallow: /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow: + /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow: + /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow: + /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow: + /*/shop/storeConfig*\nDisallow: /*/shop/vieworder*\nDisallow: /*/variationSelection*\nDisallow: + /ae/*\nDisallow: /am/*\nDisallow: /asia/*\nDisallow: /at/*\nDisallow: /au/*\nDisallow: + /befr/*\nDisallow: /benl/*\nDisallow: /bg/*\nDisallow: /bh/*\nDisallow: /br/*\nDisallow: + /bw/*\nDisallow: /ca/*\nDisallow: /ca/fr/*\nDisallow: /cf/*\nDisallow: /chde/*\nDisallow: + /chfr/*\nDisallow: /ci/*\nDisallow: /cl/*\nDisallow: /cm/*\nDisallow: /cn/se/*\nDisallow: + /co/*\nDisallow: /cz/*\nDisallow: /de/*\nDisallow: /dk/*\nDisallow: /ee/*\nDisallow: + /eg/*\nDisallow: /es/*\nDisallow: /fi/*\nDisallow: /fr/*\nDisallow: /gn/*\nDisallow: + /gq/*\nDisallow: /gr/*\nDisallow: /gw/*\nDisallow: /hk/*\nDisallow: /hk/en/*\nDisallow: + /hr/*\nDisallow: /hu/*\nDisallow: /id/*\nDisallow: /ie/*\nDisallow: /il/*\nDisallow: + /in/*\nDisallow: /it/*\nDisallow: /jo/*\nDisallow: /jp/*\nDisallow: /ke/*\nDisallow: + /kr/*\nDisallow: /kw/*\nDisallow: /la/*\nDisallow: /lae/*\nDisallow: /li/*\nDisallow: + /lt/*\nDisallow: /lu/*\nDisallow: /lv/*\nDisallow: /ma/*\nDisallow: /md/*\nDisallow: + /me/*\nDisallow: /mg/*\nDisallow: /mk/*\nDisallow: /ml/*\nDisallow: /mt/*\nDisallow: + /mu/*\nDisallow: /mx/*\nDisallow: /my/*\nDisallow: /mz/*\nDisallow: /ne/*\nDisallow: + /ng/*\nDisallow: /nl/*\nDisallow: /no/*\nDisallow: /nz/*\nDisallow: /om/*\nDisallow: + /ph/*\nDisallow: /pl/*\nDisallow: /pt/*\nDisallow: /qa/*\nDisallow: /retail/availability*\nDisallow: + /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow: + /ro/*\nDisallow: /ru/*\nDisallow: /sa/*\nDisallow: /se/*\nDisallow: /search/*\nDisallow: + /sg/*\nDisallow: /shop/*\nDisallow: /si/*\nDisallow: /sk/*\nDisallow: /sn/*\nDisallow: + /th/*\nDisallow: /tmall/*\nDisallow: /tn/*\nDisallow: /tr/*\nDisallow: /tw/*\nDisallow: + /ug/*\nDisallow: /uk/*\nDisallow: /vn/*\nDisallow: /za/*\n\nUser-agent: HaoSouSpider\nDisallow: + /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow: + /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow: + /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow: + /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow: + /*/shop/button_availability*\nDisallow: /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: + /*/shop/checkout*\nDisallow: /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: + /*/shop/identify_user*\nDisallow: /*/shop/mobile/checkout/start*\nDisallow: + /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow: + /*/shop/question/unsubscribe/*\nDisallow: /*/shop/review/vote*\nDisallow: + /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: + /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow: + /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow: + /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow: + /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow: + /*/shop/storeConfig*\nDisallow: /*/shop/variationSelection*\nDisallow: /*/shop/vieworder*\nDisallow: + /cn/se/*\nDisallow: /retail/availability*\nDisallow: /retail/availabilitySearch*\nDisallow: + /retail/pickupEligibility*\nDisallow: /search/*\nDisallow: /shop/bag*\nDisallow: + /shop/search/*\nDisallow: /tmall/*\n\nSitemap: http://www.apple.com/shop/sitemap.xml\nSitemap: + http://www.apple.com/sitemap.xml\n" + http_version: + recorded_at: Mon, 04 Jan 2016 13:39:17 GMT +- request: + method: get + uri: http://www.apple.com/robots.txt + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - TestBaiduspider + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Last-Modified: + - Thu, 22 Oct 2015 00:20:47 GMT + Server: + - Apache + Nncoection: + - close + Accept-Ranges: + - bytes + Content-Length: + - '6902' + Content-Type: + - text/plain; charset=UTF-8 + Cache-Control: + - max-age=417 + Expires: + - Mon, 04 Jan 2016 13:46:14 GMT + Date: + - Mon, 04 Jan 2016 13:39:17 GMT + Connection: + - keep-alive + body: + encoding: UTF-8 + string: "# robots.txt for http://www.apple.com/\n\nUser-agent: *\nDisallow: + /*/includes/*\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow: + /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/signed_in_account*\nDisallow: + /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/*WebObjects/*\nDisallow: + /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/answer/vote*\nDisallow: /*/shop/bag*\nDisallow: + /*/shop/browse/campaigns/mobile_overlay*\nDisallow: /*/shop/button_availability*\nDisallow: + /*/shop/favorites*\nDisallow: /*/shop/mobile/olss_error*\nDisallow: /*/shop/mobilex/*\nDisallow: + /*/shop/order/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow: + /*/shop/question/unsubscribe/\nDisallow: /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: + /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: /*/shop/socialsharing/*\nDisallow: + /*/shop/store/feeds/*\nDisallow: /*/shop/variationSelection\nDisallow: /*_adc_*/shop/\nDisallow: + /*_aoc_*/shop/\nDisallow: /*_edu_*/shop/\nDisallow: /*_enterprise*/shop/\nDisallow: + /*_internal-epp-discounted*/shop/\nDisallow: /*_k12nonbts*/shop/\nDisallow: + /*_kiosk*/shop/\nDisallow: /*_nonbts*/shop/\nDisallow: /*_qpromo*/shop/\nDisallow: + /*_refurb-discounted*/shop/\nDisallow: /cn/se/*\nDisallow: /retail/availability*\nDisallow: + /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow: + /search/*\nDisallow: /shop/*signed_in_account*\nDisallow: /shop/*sign_in*\nDisallow: + /shop/*sign_out*\nDisallow: /shop/*WebObjects/*\nDisallow: /shop/1-800-MY-APPLE/*\nDisallow: + /shop/answer/vote*\nDisallow: /shop/bag*\nDisallow: /shop/browse/campaigns/mobile_overlay*\nDisallow: + /shop/button_availability*\nDisallow: /shop/favorites*\nDisallow: /shop/mobile/olss_error*\nDisallow: + /shop/mobilex/\nDisallow: /shop/order/*\nDisallow: /shop/question/answer/report*\nDisallow: + /shop/question/subscribe*\nDisallow: /shop/question/unsubscribe/\nDisallow: + /shop/review/vote*\nDisallow: /shop/reviews/report*\nDisallow: /shop/rs-mvt/rel/*\nDisallow: + /shop/search/*\nDisallow: /shop/sentry*\nDisallow: /shop/socialsharing/*\nDisallow: + /shop/store/feeds/*\nDisallow: /shop/variationSelection*\nDisallow: /tmall*\nAllow: + /ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search/* \t\n\nUser-agent: + Baiduspider\nDisallow: /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow: + /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow: + /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow: + /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow: + /*/shop/browse/overlay*\nDisallow: /*/shop/button_availability*\nDisallow: + /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: /*/shop/checkout*\nDisallow: + /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: /*/shop/identify_user*\nDisallow: + /*/shop/mobile/checkout/start*\nDisallow: /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: + /*/shop/question/subscribe*\nDisallow: /*/shop/question/unsubscribe/*\nDisallow: + /*/shop/review/vote*\nDisallow: /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: + /*/shop/sentry*\nDisallow: /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow: + /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow: + /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow: + /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow: + /*/shop/storeConfig*\nDisallow: /*/shop/vieworder*\nDisallow: /*/variationSelection*\nDisallow: + /ae/*\nDisallow: /am/*\nDisallow: /asia/*\nDisallow: /at/*\nDisallow: /au/*\nDisallow: + /befr/*\nDisallow: /benl/*\nDisallow: /bg/*\nDisallow: /bh/*\nDisallow: /br/*\nDisallow: + /bw/*\nDisallow: /ca/*\nDisallow: /ca/fr/*\nDisallow: /cf/*\nDisallow: /chde/*\nDisallow: + /chfr/*\nDisallow: /ci/*\nDisallow: /cl/*\nDisallow: /cm/*\nDisallow: /cn/se/*\nDisallow: + /co/*\nDisallow: /cz/*\nDisallow: /de/*\nDisallow: /dk/*\nDisallow: /ee/*\nDisallow: + /eg/*\nDisallow: /es/*\nDisallow: /fi/*\nDisallow: /fr/*\nDisallow: /gn/*\nDisallow: + /gq/*\nDisallow: /gr/*\nDisallow: /gw/*\nDisallow: /hk/*\nDisallow: /hk/en/*\nDisallow: + /hr/*\nDisallow: /hu/*\nDisallow: /id/*\nDisallow: /ie/*\nDisallow: /il/*\nDisallow: + /in/*\nDisallow: /it/*\nDisallow: /jo/*\nDisallow: /jp/*\nDisallow: /ke/*\nDisallow: + /kr/*\nDisallow: /kw/*\nDisallow: /la/*\nDisallow: /lae/*\nDisallow: /li/*\nDisallow: + /lt/*\nDisallow: /lu/*\nDisallow: /lv/*\nDisallow: /ma/*\nDisallow: /md/*\nDisallow: + /me/*\nDisallow: /mg/*\nDisallow: /mk/*\nDisallow: /ml/*\nDisallow: /mt/*\nDisallow: + /mu/*\nDisallow: /mx/*\nDisallow: /my/*\nDisallow: /mz/*\nDisallow: /ne/*\nDisallow: + /ng/*\nDisallow: /nl/*\nDisallow: /no/*\nDisallow: /nz/*\nDisallow: /om/*\nDisallow: + /ph/*\nDisallow: /pl/*\nDisallow: /pt/*\nDisallow: /qa/*\nDisallow: /retail/availability*\nDisallow: + /retail/availabilitySearch*\nDisallow: /retail/pickupEligibility*\nDisallow: + /ro/*\nDisallow: /ru/*\nDisallow: /sa/*\nDisallow: /se/*\nDisallow: /search/*\nDisallow: + /sg/*\nDisallow: /shop/*\nDisallow: /si/*\nDisallow: /sk/*\nDisallow: /sn/*\nDisallow: + /th/*\nDisallow: /tmall/*\nDisallow: /tn/*\nDisallow: /tr/*\nDisallow: /tw/*\nDisallow: + /ug/*\nDisallow: /uk/*\nDisallow: /vn/*\nDisallow: /za/*\n\nUser-agent: HaoSouSpider\nDisallow: + /*/retail/availability*\nDisallow: /*/retail/availabilitySearch*\nDisallow: + /*/retail/pickupEligibility*\nDisallow: /*/search/*\nDisallow: /*/shop/*WebObjects/*\nDisallow: + /*/shop/1-800-MY-APPLE/*\nDisallow: /*/shop/2322-consumer*\nDisallow: /*/shop/account/setup*\nDisallow: + /*/shop/answer/vote*\nDisallow: /*/shop/browse/campaigns/mobile_overlay*\nDisallow: + /*/shop/button_availability*\nDisallow: /*/shop/bag*\nDisallow: /*/shop/change_password*\nDisallow: + /*/shop/checkout*\nDisallow: /*/shop/create_account*\nDisallow: /*/shop/favorites*\nDisallow: + /*/shop/identify_user*\nDisallow: /*/shop/mobile/checkout/start*\nDisallow: + /*/shop/mobilex/*\nDisallow: /*/shop/question/answer/report*\nDisallow: /*/shop/question/subscribe*\nDisallow: + /*/shop/question/unsubscribe/*\nDisallow: /*/shop/review/vote*\nDisallow: + /*/shop/reviews/report*\nDisallow: /*/shop/rs-mvt/rel/*\nDisallow: /*/shop/sentry*\nDisallow: + /*/shop/sentryx/change_password*\nDisallow: /*/shop/sentryx/create_account*\nDisallow: + /*/shop/sentryx/create_account_confirm*\nDisallow: /*/shop/sentryx/identify_user*\nDisallow: + /*/shop/sentryx/sign_in*\nDisallow: /*/shop/signed_in_account*\nDisallow: + /*/shop/sign_in*\nDisallow: /*/shop/sign_out*\nDisallow: /*/shop/socialsharing/*\nDisallow: + /*/shop/storeConfig*\nDisallow: /*/shop/variationSelection*\nDisallow: /*/shop/vieworder*\nDisallow: + /cn/se/*\nDisallow: /retail/availability*\nDisallow: /retail/availabilitySearch*\nDisallow: + /retail/pickupEligibility*\nDisallow: /search/*\nDisallow: /shop/bag*\nDisallow: + /shop/search/*\nDisallow: /tmall/*\n\nSitemap: http://www.apple.com/shop/sitemap.xml\nSitemap: + http://www.apple.com/sitemap.xml\n" + http_version: + recorded_at: Mon, 04 Jan 2016 13:39:17 GMT +- request: + method: get + uri: https://github.com/robots.txt + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - TestBot + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Server: + - GitHub.com + Date: + - Mon, 04 Jan 2016 13:39:18 GMT + Content-Type: + - text/plain + Last-Modified: + - Mon, 04 Jan 2016 13:28:38 GMT + Transfer-Encoding: + - chunked + Etag: + - W/"568a7386-6e36" + Vary: + - Accept-Encoding + X-Served-By: + - a568c03544f42dddf712bab3bfd562fd + X-Github-Request-Id: + - 7C27244A:448A:3EB799EF:568A7605 + body: + encoding: ASCII-8BIT + string: | + # If you would like to crawl GitHub contact us at support@github.com. + # We also provide an extensive API: https://developer.github.com/ + + User-agent: CCBot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: coccoc + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: dotbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: duckduckbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: EtaoSpider + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Googlebot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: HTTrack + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: ia_archiver + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: IntuitGSACrawler + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Mail.RU_Bot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: msnbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Bingbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: naverbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: red-app-gsa-p-one + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: rogerbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: SandDollar + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: seznambot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Slurp + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Swiftbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Telefonica + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: teoma + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Twitterbot + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + User-agent: Yandex + Allow: /*/*/tree/master + Allow: /*/*/blob/master + Disallow: /ekansa/Open-Context-Data + Disallow: /ekansa/opencontext-* + Disallow: /*/*/pulse + Disallow: /*/*/tree/* + Disallow: /*/*/blob/* + Disallow: /*/*/wiki/*/* + Disallow: /gist/*/*/* + Disallow: /oembed + Disallow: /*/forks + Disallow: /*/stars + Disallow: /*/download + Disallow: /*/revisions + Disallow: /*/*/issues/new + Disallow: /*/*/issues/search + Disallow: /*/*/commits/*/* + Disallow: /*/*/commits/*?author + Disallow: /*/*/commits/*?path + Disallow: /*/*/branches + Disallow: /*/*/tags + Disallow: /*/*/contributors + Disallow: /*/*/comments + Disallow: /*/*/stargazers + Disallow: /*/*/search + Disallow: /*/tarball/ + Disallow: /*/zipball/ + Disallow: /*/*/archive/ + Disallow: /raw/* + Disallow: /*/followers + Disallow: /*/following + Disallow: /stars/* + Disallow: /*/blame/ + Disallow: /*/watchers + Disallow: /*/network + Disallow: /*/graphs + Disallow: /*/raw/ + Disallow: /*/compare/ + Disallow: /*/cache/ + Disallow: /*/*/blame/ + Disallow: /*/*/watchers + Disallow: /*/*/network + Disallow: /*/*/graphs + Disallow: /*/*/raw/ + Disallow: /*/*/compare/ + Disallow: /*/*/cache/ + Disallow: /*.git + Disallow: /*.git/ + Disallow: /search/advanced + Disallow: /search + Disallow: /*q= + Disallow: /*.atom + Disallow: /login + + + User-agent: * + Allow: /humans.txt + Disallow: / + http_version: + recorded_at: Mon, 04 Jan 2016 13:39:18 GMT +recorded_with: VCR 3.0.0 diff --git a/webrobots.gemspec b/webrobots.gemspec index 67e043e..06b8c55 100644 --- a/webrobots.gemspec +++ b/webrobots.gemspec @@ -29,7 +29,10 @@ This library helps write robots.txt compliant web robots in Ruby. s.add_development_dependency("rake", [">= 0.9.2.2"]) s.add_development_dependency("racc", [">= 0"]) unless RUBY_PLATFORM == "java" + s.add_development_dependency("test-unit") s.add_development_dependency("shoulda", [RUBY_VERSION < "1.9" ? "< 3.5.0" : ">= 0"]) + s.add_development_dependency("webmock") + s.add_development_dependency("vcr") if RUBY_VERSION < "1.9" # Cap dependency on activesupport with < 4.0 on behalf of # shoulda-matchers to satisfy bundler.