Permalink
Browse files

Merge pull request #4 from postrank/master

empty disallow is allow
  • Loading branch information...
temoto committed May 31, 2011
2 parents bd37b19 + db37b68 commit 7ceedec0c69db12875cb42c98495a7592c1d18a8
Showing with 23 additions and 1 deletion.
  1. +7 −1 parser.go
  2. +16 −0 robotstxt_test.go
View
@@ -67,7 +67,13 @@ func (p *Parser) ParseRule() (r *Rule, err os.Error) {
return nil, os.NewError("Disallow before User-agent.")
}
p.popToken()
- return &Rule{Agent: p.agent, Uri: t2, Allow: false}, nil
+
+ if t2 == "" {
+ return &Rule{Agent: p.agent, Uri: t2, Allow: true}, nil
+ } else {
+ return &Rule{Agent: p.agent, Uri: t2, Allow: false}, nil
+ }
+
case "allow":
if p.agent == "" {
// TODO: report error
View
@@ -184,6 +184,22 @@ func TestFromString007(t *testing.T) {
}
}
+const robots_text_004 = "User-Agent: * \nDisallow: "
+
+func TestFromString008(t *testing.T) {
+ r, err := FromString(robots_text_004, true)
+ if err != nil {
+ t.Fatal(err.String())
+ }
+ allow, err1 := r.TestAgent("/random", "SomeBot")
+ if err1 != nil {
+ t.Fatal(err1.String())
+ }
+ if !allow {
+ t.Fatal("Must allow.")
+ }
+}
+
func BenchmarkParseFromString001(b *testing.B) {
for i := 0; i < b.N; i++ {
FromString(robots_text_001, false)

0 comments on commit 7ceedec

Please sign in to comment.