Merge pull request #14 from hazanasec/master

Add random useragent
mhmdiaa · Dec 29, 2022 · 242569b · 242569b
2 parents 4423b5d + d091b4b
commit 242569b
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 5 deletions.
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -0,0 +1,15 @@
+on:
+  release:
+    types: [created]
+
+jobs:
+  release-linux-amd64:
+    name: release linux/amd64
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: wangyoucao577/go-release-action@v1.34
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        goos: linux
+        goarch: amd64
diff --git a/second-order.go b/second-order.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -125,6 +126,11 @@ func main() {
 
 	// Add headers
 	c.OnRequest(func(r *colly.Request) {
+		// Set a random user agent for each request
+		rand.Seed(time.Now().Unix())
+		n := rand.Intn(len(userAgents))
+		r.Headers.Set("User-Agent", userAgents[n])
+		// Add other headers
 		for header, value := range headers {
 			r.Headers.Set(header, value)
 		}
@@ -209,19 +215,19 @@ func writeAllResults(config Configuration) {
 	os.MkdirAll(outdir, os.ModePerm)
 
 	if config.LogQueries != nil {
-		err := writeResults("attributes.json", loggedQueries.content)
+		err := writeResults("attributes.json", loggedQueries.content, "LogQueries")
 		if err != nil {
 			log.Printf("Error writing attributes: %v", err)
 		}
 	}
 	if config.LogInline != nil {
-		err := writeResults("inline.json", loggedInline.content)
+		err := writeResults("inline.json", loggedInline.content, "LogInline")
 		if err != nil {
 			log.Printf("Error writing inline text: %v", err)
 		}
 	}
 	if config.LogNon200Queries != nil {
-		err := writeResults("non-200-url-attributes.json", loggedNon200Queries.content)
+		err := writeResults("non-200-url-attributes.json", loggedNon200Queries.content, "LogNon200Queries")
 		if err != nil {
 			log.Printf("Error writing non-200 URL attributes: %v", err)
 		}
@@ -258,8 +264,10 @@ func unpackQuerySelector(q string) (string, string) {
 	return tag, attribute
 }
 
-func writeResults(filename string, content map[string]map[string][]string) error {
-	JSON, err := json.Marshal(content)
+func writeResults(filename string, content map[string]map[string][]string, resultType string) error {
+	output := make(map[string]map[string]map[string][]string)
+	output[resultType] = content
+	JSON, err := json.Marshal(output)
 	if err != nil {
 		return fmt.Errorf("could not marshal the JSON object: %v", err)
 	}
@@ -336,3 +344,30 @@ func isNotFound(url string) bool {
 	}
 	return false
 }
+
+var userAgents = []string{
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.54",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.46",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.67",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0; rv:87.0) Gecko/20100101 Firefox/87.0",
+	"Mozilla/5.0 (X11; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/87.0.664.59",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/87.0.664.56",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0",
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0; rv:85.0) Gecko/20100101 Firefox/85.0",
+	"Mozilla/5.0 (X11; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.72",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36",
+}