Decode robots.txt response content before attempting to parse it.

gisle · May 3, 2011 · 83be3c5 · 83be3c5
1 parent 38ea477
commit 83be3c5
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/lib/LWP/RobotUA.pm b/lib/LWP/RobotUA.pm
@@ -129,8 +129,8 @@ sub simple_request
 	my $robot_res = $self->request($robot_req);
 	my $fresh_until = $robot_res->fresh_until;
 	if ($robot_res->is_success && $robot_res->content_is_text) {
-	    my $c = $robot_res->content;
-	    if ($c =~ /^\s*Disallow\s*:/mi) {
+	    my $c = $robot_res->decoded_content;
+	    if ($c && $c =~ /^\s*Disallow\s*:/mi) {
 		$self->{'rules'}->parse($robot_url, $c, $fresh_until);
 	    }
 	    else {