Permalink
Browse files

Fixes #10 FindGroup should never return nil

  • Loading branch information...
1 parent 164bac4 commit 961a45f5584786f2c07357125295f6d0f985e209 @temoto committed Nov 26, 2012
Showing with 26 additions and 0 deletions.
  1. +5 −0 robotstxt.go
  2. +21 −0 robotstxt_test.go
View
5 robotstxt.go
@@ -57,6 +57,7 @@ func (e ParseError) Error() string {
var allowAll = &RobotsData{allowAll: true}
var disallowAll = &RobotsData{disallowAll: true}
+var emptyGroup = &Group{}
func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) {
switch {
@@ -182,6 +183,10 @@ func (r *RobotsData) FindGroup(agent string) (ret *Group) {
}
}
}
+
+ if ret == nil {
+ return emptyGroup
+ }
return
}
View
21 robotstxt_test.go
@@ -211,6 +211,27 @@ func TestParseErrors(t *testing.T) {
}
}
+const robots_text_just_html = `<!DOCTYPE html>
+<html>
+<title></title>
+<p>Hello world! This is valid HTML but invalid robots.txt.`
+
+func TestHtmlInstead(t *testing.T) {
+ r, err := FromString(robots_text_just_html)
+ if err != nil {
+ // According to Google spec, invalid robots.txt file
+ // must be parsed silently.
+ t.Fatal(err.Error())
+ }
+ group := r.FindGroup("SuperBot")
+ if group == nil {
+ t.Fatal("Group must not be nil.")
+ }
+ if !group.Test("/") {
+ t.Fatal("Must allow by default.")
+ }
+}
+
func BenchmarkParseFromString001(b *testing.B) {
for i := 0; i < b.N; i++ {
FromString(robots_text_001)

0 comments on commit 961a45f

Please sign in to comment.