Skip to content
Permalink
Browse files

added flags max_request_retries, request_retry_timeout, sleep_time

  • Loading branch information...
marcobeierer committed Sep 2, 2019
1 parent 74ccad2 commit 1d0b76dc8856aeebd9c35bfa5f4484a1710d7804
Showing with 20 additions and 6 deletions.
  1. +6 −0 README.md
  2. +14 −6 main.go
@@ -23,6 +23,12 @@ The sitemap is written to the standard output. It is thus possible to redirect t
- Number of the maximal concurrent connections.
- reference\_count\_threshold
- With the reference count threshold you can define that images and videos that are embedded on more than the selected number of HTML pages are excluded from the sitemap.
- max\_request\_retries
- Number of retries for each failed request
- request\_retry\_timeout
- Timeout in seconds after a failed request
- sleep\_time
- Seconds between each update request

### Example
sitemapgenerator -tokenpath token.txt https://www.marcobeierer.com > sitemap.xml
20 main.go
@@ -22,6 +22,10 @@ func main() {
referenceCountThreshold := flag.Int64("reference_count_threshold", -1, "With the reference count threshold you can define that images and videos that are embedded on more than the selected number of HTML pages are excluded from the sitemap.")
maxFetchers := flag.Int64("max_fetchers", 3, "Number of the maximal concurrent connections.")

maxRequestRetries := flag.Int64("max_request_retries", 5, "Number of retries for each failed request")
requestRetryTimeoutInSeconds := flag.Int64("request_retry_timeout", 30, "Timeout in seconds after a failed request")
sleepTimeInSeconds := flag.Int64("sleep_time", 5, "Seconds between each update request")

flag.Parse()

token, ok := readToken(*tokenPath)
@@ -36,10 +40,10 @@ func main() {
return
}

retries := 0
retriesCount := int64(0)
for {
if body, statusCode, contentType, stats, limitReached, ok := doRequest(url, token, *maxFetchers, *referenceCountThreshold); ok {
retries = 0 // always reset retries count on a successfull request
retriesCount = 0 // always reset retries count on a successfull request

if contentType == "application/xml" {
if stats != "" {
@@ -51,20 +55,24 @@ func main() {
fmt.Println(body)
return
}
} else if statusCode == 0 && retries < 3 {
} else if statusCode == 0 && retriesCount < *maxRequestRetries {
// do up to three retries if request fails
// the easiest way to simulate retries is to add an invalid port the sitemap generator API URL (api.marcobeierer.com) below
retries++
retriesCount++

// sleep a little longer if there was an error, might be a refused connection due to too much requests in short time
time.Sleep(time.Duration(*requestRetryTimeoutInSeconds) * time.Second)

// don't `continue` because we want to sleep anyway
} else {
if retries > 0 {
if retriesCount > 0 {
log.Fatalln("multiple request failed, abort sitemap generation")
} else {
log.Fatalln("request failed, abort sitemap generation")
}
return
}
time.Sleep(5 * time.Second)
time.Sleep(time.Duration(*sleepTimeInSeconds) * time.Second)
}
}

0 comments on commit 1d0b76d

Please sign in to comment.
You can’t perform that action at this time.