Permalink
Browse files

Don't parse robots.txt response content unless it's textual.

  • Loading branch information...
1 parent 34ae651 commit 38ea477b1c63787b0766a1e828ea68117a86454d @scop scop committed Apr 30, 2011
Showing with 2 additions and 2 deletions.
  1. +2 −2 lib/LWP/RobotUA.pm
View
@@ -128,9 +128,9 @@ sub simple_request
my $robot_req = HTTP::Request->new('GET', $robot_url);
my $robot_res = $self->request($robot_req);
my $fresh_until = $robot_res->fresh_until;
- if ($robot_res->is_success) {
+ if ($robot_res->is_success && $robot_res->content_is_text) {
my $c = $robot_res->content;
- if ($robot_res->content_type =~ m,^text/, && $c =~ /^\s*Disallow\s*:/mi) {
+ if ($c =~ /^\s*Disallow\s*:/mi) {
$self->{'rules'}->parse($robot_url, $c, $fresh_until);
}
else {

0 comments on commit 38ea477

Please sign in to comment.