chore: -

hueristiq · Jul 26, 2023 · d0bcea6 · d0bcea6
1 parent c006d8e
commit d0bcea6
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 12 deletions.
diff --git a/pkg/xurlfind3r/httpclient/client.go b/pkg/xurlfind3r/httpclient/client.go
@@ -15,10 +15,6 @@ func httpRequestWrapper(req *fasthttp.Request) (res *fasthttp.Response, err erro
 		return
 	}
 
-	// if res.StatusCode() != fasthttp.StatusOK {
-	// 	err = fmt.Errorf("unexpected status code")
-	// }
-
 	return
 }
 

diff --git a/pkg/xurlfind3r/sources/bevigil/bevigil.go b/pkg/xurlfind3r/sources/bevigil/bevigil.go
@@ -22,9 +22,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 	go func() {
 		defer close(URLsChannel)
 
-		var key string
 		var err error
 
+		var key string
+
 		key, err = sources.PickRandom(config.Keys.Bevigil)
 		if key == "" || err != nil {
 			return

diff --git a/pkg/xurlfind3r/sources/commoncrawl/commoncrawl.go b/pkg/xurlfind3r/sources/commoncrawl/commoncrawl.go
@@ -31,9 +31,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 	go func() {
 		defer close(URLsChannel)
 
-		var indexesRes *fasthttp.Response
 		var err error
 
+		var indexesRes *fasthttp.Response
+
 		indexesRes, err = httpclient.SimpleGet("https://index.commoncrawl.org/collinfo.json")
 		if err != nil {
 			return
@@ -57,9 +58,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 					"Host": "index.commoncrawl.org",
 				}
 
-				var contentRes *fasthttp.Response
 				var err error
 
+				var contentRes *fasthttp.Response
+
 				contentRes, err = httpclient.Get(fmt.Sprintf("%s?url=*.%s/*&output=json&fl=url", API, domain), "", contentReqHeaders)
 				if err != nil {
 					return

diff --git a/pkg/xurlfind3r/sources/github/github.go b/pkg/xurlfind3r/sources/github/github.go
@@ -51,7 +51,7 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 	return URLsChannel
 }
 
-func (source *Source) Enumerate(searchURL string, domain string, tokens *Tokens, URLsChannel chan sources.URL, config *sources.Configuration) {
+func (source *Source) Enumerate(searchURL, domain string, tokens *Tokens, URLsChannel chan sources.URL, config *sources.Configuration) {
 	token := tokens.Get()
 
 	if token.RetryAfter > 0 {
@@ -68,10 +68,13 @@ func (source *Source) Enumerate(searchURL string, domain string, tokens *Tokens,
 	}
 
 	var err error
+
 	var searchRes *fasthttp.Response
 
 	searchRes, err = httpclient.Request(fasthttp.MethodGet, searchURL, "", reqHeaders, nil)
+
 	isForbidden := searchRes != nil && searchRes.StatusCode() == fasthttp.StatusForbidden
+
 	if err != nil && !isForbidden {
 		return
 	}

diff --git a/pkg/xurlfind3r/sources/intelx/intelx.go b/pkg/xurlfind3r/sources/intelx/intelx.go
@@ -39,9 +39,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 	go func() {
 		defer close(URLsChannel)
 
-		var key string
 		var err error
 
+		var key string
+
 		key, err = sources.PickRandom(config.Keys.Intelx)
 		if key == "" || err != nil {
 			return

diff --git a/pkg/xurlfind3r/sources/otx/otx.go b/pkg/xurlfind3r/sources/otx/otx.go
@@ -38,9 +38,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 		for page := 1; ; page++ {
 			reqURL := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/url_list?limit=%d&page=%d", domain, 200, page)
 
-			var res *fasthttp.Response
 			var err error
 
+			var res *fasthttp.Response
+
 			res, err = httpclient.SimpleGet(reqURL)
 			if err != nil {
 				return

diff --git a/pkg/xurlfind3r/sources/urlscan/urlscan.go b/pkg/xurlfind3r/sources/urlscan/urlscan.go
@@ -35,9 +35,10 @@ func (source *Source) Run(config *sources.Configuration, domain string) (URLsCha
 	go func() {
 		defer close(URLsChannel)
 
-		var key string
 		var err error
 
+		var key string
+
 		key, err = sources.PickRandom(config.Keys.URLScan)
 		if err != nil {
 			return

diff --git a/pkg/xurlfind3r/sources/wayback/waybackrobots.go b/pkg/xurlfind3r/sources/wayback/waybackrobots.go
@@ -10,7 +10,7 @@ import (
 	"github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/sources"
 )
 
-func parseWaybackRobots(config *sources.Configuration, URL string) (robotsURLs chan string) {
+func parseWaybackRobots(_ *sources.Configuration, URL string) (robotsURLs chan string) {
 	robotsURLs = make(chan string)
 
 	robotsEntryRegex := regexp.MustCompile(`Disallow:\s?.+`)

diff --git a/pkg/xurlfind3r/sources/wayback/waybacksource.go b/pkg/xurlfind3r/sources/wayback/waybacksource.go
@@ -17,6 +17,7 @@ func parseWaybackSource(domain, URL string) (sourceURLs chan string) {
 		defer close(sourceURLs)
 
 		var err error
+
 		var snapshots [][2]string
 
 		snapshots, err = getWaybackSnapshots(URL)