Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Don't parse robots.txt response content unless it's textual.

  • Loading branch information...
commit 38ea477b1c63787b0766a1e828ea68117a86454d 1 parent 34ae651
Ville Skyttä scop authored
Showing with 2 additions and 2 deletions.
  1. +2 −2 lib/LWP/RobotUA.pm
4 lib/LWP/RobotUA.pm
View
@@ -128,9 +128,9 @@ sub simple_request
my $robot_req = HTTP::Request->new('GET', $robot_url);
my $robot_res = $self->request($robot_req);
my $fresh_until = $robot_res->fresh_until;
- if ($robot_res->is_success) {
+ if ($robot_res->is_success && $robot_res->content_is_text) {
my $c = $robot_res->content;
- if ($robot_res->content_type =~ m,^text/, && $c =~ /^\s*Disallow\s*:/mi) {
+ if ($c =~ /^\s*Disallow\s*:/mi) {
$self->{'rules'}->parse($robot_url, $c, $fresh_until);
}
else {
Please sign in to comment.
Something went wrong with that request. Please try again.