block more go mirrors

dash/spider.go

@@ -29,7 +29,7 @@ import (
 )
 
 const (
-	spiderVersion = 1 // increment this when we change the spidering algorithm
+	spiderVersion = 2 // increment this when we change the spidering algorithm
 	spiderQueue   = "spider"
 )
 
@@ -219,6 +219,11 @@ func gopkgdocPaths(c appengine.Context) ([]string, error) {
 }
 
 func pathOK(p string) bool {
-	return !(doc.StandardPackages[p] ||
-		strings.HasPrefix(p, "code.google.com/p/go/"))
+	if doc.StandardPackages[p] ||
+		strings.HasPrefix(p, "code.google.com/p/go/") ||
+		strings.Contains(p, "/src/pkg/go/") || // Block Go mirrors.
+		strings.Contains(p, "/src/cmd/go/") {
+		return false
+	}
+	return true
 }