refactor: scope page counter to pagination loop

lc · Jul 14, 2023 · 04528bb · 04528bb
1 parent 674f69e
commit 04528bb
Showing 1 changed file with 4 additions and 6 deletions.
diff --git a/pkg/providers/urlscan/urlscan.go b/pkg/providers/urlscan/urlscan.go
@@ -41,9 +41,8 @@ func (c *Client) Fetch(ctx context.Context, domain string, results chan string)
 		header.Value = c.config.URLScan.APIKey
 	}
 
-	page := 0
 paginate:
-	for {
+	for page := 0; ; page++ {
 		select {
 		case <-ctx.Done():
 			break paginate
@@ -65,7 +64,7 @@ paginate:
 			// rate limited
 			if result.Status == 429 {
 				if c.config.Verbose {
-					logrus.WithField("provider", "urlscan").Warnf("urlscan responded with 429")
+					logrus.WithField("provider", "urlscan").Warnf("urlscan responded with 429, probably being rate limited")
 				}
 				break paginate
 			}
@@ -89,18 +88,17 @@ paginate:
 			if !result.HasMore {
 				break paginate
 			}
-			page++
 		}
 	}
 	return nil
 }
 
 func (c *Client) formatURL(domain string, after string) string {
 	if after != "" {
-		return fmt.Sprintf(_BaseURL+"api/v1/search/?q=domain:%s&size=100", domain) + "&search_after=" + after
+		after = "&search_after=" + after
 	}
 
-	return fmt.Sprintf(_BaseURL+"api/v1/search/?q=domain:%s&size=100", domain)
+	return fmt.Sprintf(_BaseURL+"api/v1/search/?q=domain:%s&size=100", domain) + after
 }
 
 func setBaseURL(baseURL string) {