From 1727854d05f881adede433ac451aca9b15e42de5 Mon Sep 17 00:00:00 2001 From: thewizardplusplus Date: Sun, 17 Jul 2022 01:21:23 +0300 Subject: [PATCH 1/2] Make the `AllowAll` instance public --- robotstxt.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/robotstxt.go b/robotstxt.go index 52d3637..a483b77 100644 --- a/robotstxt.go +++ b/robotstxt.go @@ -56,7 +56,7 @@ func (e ParseError) Error() string { return b.String() } -var allowAll = &RobotsData{allowAll: true} +var AllowAll = &RobotsData{allowAll: true} var disallowAll = &RobotsData{disallowAll: true} var emptyGroup = &Group{} @@ -72,7 +72,7 @@ func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) { // This is a "full allow" for crawling. Note: this includes 401 // "Unauthorized" and 403 "Forbidden" HTTP result codes. case statusCode >= 400 && statusCode < 500: - return allowAll, nil + return AllowAll, nil // From Google's spec: // Server errors (5xx) are seen as temporary errors that result in a "full @@ -106,7 +106,7 @@ func FromBytes(body []byte) (r *RobotsData, err error) { // special case (probably not worth optimization?) trimmed := bytes.TrimSpace(body) if len(trimmed) == 0 { - return allowAll, nil + return AllowAll, nil } sc := newByteScanner("bytes", true) @@ -116,7 +116,7 @@ func FromBytes(body []byte) (r *RobotsData, err error) { // special case worth optimization if len(tokens) == 0 { - return allowAll, nil + return AllowAll, nil } r = &RobotsData{} From 60a6ab825cf6041f932be722eded848656aa7a7b Mon Sep 17 00:00:00 2001 From: thewizardplusplus Date: Sun, 17 Jul 2022 01:26:49 +0300 Subject: [PATCH 2/2] Make the `DisallowAll` instance public --- robotstxt.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/robotstxt.go b/robotstxt.go index a483b77..8aea38c 100644 --- a/robotstxt.go +++ b/robotstxt.go @@ -57,7 +57,7 @@ func (e ParseError) Error() string { } var AllowAll = &RobotsData{allowAll: true} -var disallowAll = &RobotsData{disallowAll: true} +var DisallowAll = &RobotsData{disallowAll: true} var emptyGroup = &Group{} func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) { @@ -78,7 +78,7 @@ func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) { // Server errors (5xx) are seen as temporary errors that result in a "full // disallow" of crawling. case statusCode >= 500 && statusCode < 600: - return disallowAll, nil + return DisallowAll, nil } return nil, errors.New("Unexpected status: " + strconv.Itoa(statusCode))