Imporve handling of testdata directories

Move the "testdata" directory filter from gosrc.IsValidRemotePath to the
crawler. This fixes the incorrect error in #250.

gddo-server/crawl.go

@@ -8,13 +8,16 @@ package main
 
 import (
 	"log"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/golang/gddo/doc"
 	"github.com/golang/gddo/gosrc"
 )
 
+var testdataPat = regexp.MustCompile(`/testdata(?:/|$)`)
+
 // crawlDoc fetches the package documentation from the VCS and updates the database.
 func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bool, nextCrawl time.Time) (*doc.Package, error) {
 	message := []interface{}{source}
@@ -45,6 +48,9 @@ func crawlDoc(source string, importPath string, pdoc *doc.Package, hasSubdirs bo
 	} else if blocked, e := db.IsBlocked(importPath); blocked && e == nil {
 		pdoc = nil
 		err = gosrc.NotFoundError{Message: "blocked."}
+	} else if testdataPat.MatchString(importPath) {
+		pdoc = nil
+		err = gosrc.NotFoundError{Message: "testdata."}
 	} else {
 		var pdocNew *doc.Package
 		pdocNew, err = doc.Get(httpClient, importPath, etag)

gosrc/path.go

@@ -18,7 +18,7 @@ var validHost = regexp.MustCompile(`^[-a-z0-9]+(?:\.[-a-z0-9]+)+$`)
 var validPathElement = regexp.MustCompile(`^[-A-Za-z0-9~+_][-A-Za-z0-9_.]*$`)
 
 func isValidPathElement(s string) bool {
-	return validPathElement.MatchString(s) && s != "testdata"
+	return validPathElement.MatchString(s)
 }
 
 // IsValidRemotePath returns true if importPath is structurally valid for "go get".

gosrc/path_test.go

@@ -28,7 +28,6 @@ var badImportPaths = []string{
 	".bar",
 	"favicon.ico",
 	"exmpple.com",
-	"github.com/user/repo/testdata/x",
 	"github.com/user/repo/.ignore/x",
 }