-
-
Notifications
You must be signed in to change notification settings - Fork 423
/
urlscan.go
108 lines (93 loc) · 2.37 KB
/
urlscan.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
package urlscan
import (
"bytes"
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/lc/gau/v2/pkg/httpclient"
"github.com/lc/gau/v2/pkg/providers"
"github.com/sirupsen/logrus"
"strings"
)
const (
Name = "urlscan"
)
var _ providers.Provider = (*Client)(nil)
type Client struct {
config *providers.Config
}
func New(c *providers.Config) *Client {
if c.URLScan.Host != "" {
setBaseURL(c.URLScan.Host)
}
return &Client{config: c}
}
func (c *Client) Name() string {
return Name
}
func (c *Client) Fetch(ctx context.Context, domain string, results chan string) error {
var searchAfter string
var header httpclient.Header
if c.config.URLScan.APIKey != "" {
header.Key = "API-Key"
header.Value = c.config.URLScan.APIKey
}
page := 0
paginate:
for {
select {
case <-ctx.Done():
break paginate
default:
if c.config.Verbose {
logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
}
apiURL := c.formatURL(domain, searchAfter)
resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout, header)
if err != nil {
return fmt.Errorf("failed to fetch urlscan: %s", err)
}
var result apiResponse
decoder := jsoniter.NewDecoder(bytes.NewReader(resp))
decoder.UseNumber()
if err = decoder.Decode(&result); err != nil {
return fmt.Errorf("failed to decode urlscan result: %s", err)
}
// rate limited
if result.Status == 429 {
if c.config.Verbose {
logrus.WithField("provider", "urlscan").Warnf("urlscan responded with 429")
}
break paginate
}
total := len(result.Results)
for i, res := range result.Results {
if res.Page.Domain == domain || (c.config.IncludeSubdomains && strings.HasSuffix(res.Page.Domain, domain)) {
results <- res.Page.URL
}
if i == total-1 {
sortParam := parseSort(res.Sort)
if sortParam != "" {
searchAfter = sortParam
} else {
break paginate
}
}
}
if !result.HasMore {
break paginate
}
page++
}
}
return nil
}
func (c *Client) formatURL(domain string, after string) string {
if after != "" {
return fmt.Sprintf(_BaseURL+"api/v1/search/?q=domain:%s&size=100", domain) + "&search_after=" + after
}
return fmt.Sprintf(_BaseURL+"api/v1/search/?q=domain:%s&size=100", domain)
}
func setBaseURL(baseURL string) {
_BaseURL = baseURL
}