diff --git a/README.md b/README.md index 4611dd778..6392ae2f8 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping. Additionally, Amass uses the IP addresses obtained during resolution to discover associated netblocks and ASNs. All the information is then used to build maps of the target networks. -**Information gathering techniques used:** +**Information Gathering Techniques Used:** * DNS: Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request) * Scraping: Ask, Baidu, Bing, CommonCrawl, DNSDB, DNSDumpster, DNSTable, Dogpile, Exalead, FindSubdomains, Google, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ThreatCrowd, VirusTotal, Yahoo @@ -44,6 +44,22 @@ If your operating environment supports [Snap](https://docs.snapcraft.io/core/ins sudo snap install amass ``` +On Kali, follow these steps to install Snap and Amass + use AppArmor (for autoload): + +```bash +sudo apt install snapd +sudo systemctl start snapd +sudo systemctl enable snapd +sudo systemctl start apparmor +sudo systemctl enable apparmor +``` + +Add the Snap bin directory to your PATH: + +```bash +export PATH=$PATH:/snap/bin +``` + Periodically, execute the following command to update all your snap packages: ```bash @@ -121,6 +137,8 @@ This project improves thanks to all the people who contribute: [![Follow on Twitter](https://img.shields.io/twitter/follow/rbadguy1.svg?logo=twitter)](https://twitter.com/rbadguy1) [![Follow on Twitter](https://img.shields.io/twitter/follow/adam_zinger.svg?logo=twitter)](https://twitter.com/adam_zinger) [![Follow on Twitter](https://img.shields.io/twitter/follow/architekton1.svg?logo=twitter)](https://twitter.com/architekton1) +[![Follow on Twitter](https://img.shields.io/twitter/follow/danjomart.svg?logo=twitter)](https://twitter.com/danjomart) +[![Follow on Twitter](https://img.shields.io/twitter/follow/_b3nj4m1n__.svg?logo=twitter)](https://twitter.com/_b3nj4m1n__) ## Mentions diff --git a/amass/amass.go b/amass/amass.go index dd4963dd3..e14565641 100644 --- a/amass/amass.go +++ b/amass/amass.go @@ -75,6 +75,7 @@ type Enumeration struct { Done chan struct{} dataSources []core.Service + bruteSrv core.Service // Pause/Resume channels for halting the enumeration pause chan struct{} @@ -155,8 +156,8 @@ func (e *Enumeration) Start() error { namesrv.RegisterGraph(e.Graph) services = append(services, namesrv, NewAddressService(e.Config, e.Bus)) if !e.Config.Passive { - services = append(services, NewAlterationService(e.Config, e.Bus), - NewBruteForceService(e.Config, e.Bus)) + e.bruteSrv = NewBruteForceService(e.Config, e.Bus) + services = append(services, NewAlterationService(e.Config, e.Bus), e.bruteSrv) } // Grab all the data sources @@ -169,12 +170,17 @@ func (e *Enumeration) Start() error { // Use all previously discovered names that are in scope go e.submitKnownNames() + // Start with the first domain name provided by the configuration + var domainIdx int + e.releaseDomainName(domainIdx) var wg sync.WaitGroup wg.Add(2) go e.checkForOutput(&wg) go e.processOutput(&wg) - t := time.NewTicker(3 * time.Second) + + tickSeconds := 3 + t := time.NewTicker(time.Duration(3) * time.Second) logTick := time.NewTicker(time.Minute) defer logTick.Stop() loop: @@ -185,9 +191,12 @@ loop: case <-e.PauseChan(): t.Stop() case <-e.ResumeChan(): - t = time.NewTicker(3 * time.Second) + t = time.NewTicker(time.Duration(3) * time.Second) case <-logTick.C: - e.processMetrics(services) + if !e.Config.Passive { + e.Config.Log.Printf("Average DNS queries performed: %d/sec, DNS names remaining: %d", + e.DNSQueriesPerSec(), e.DNSNamesRemaining()) + } case <-t.C: done := true for _, srv := range services { @@ -198,7 +207,20 @@ loop: } if done { close(e.Done) + continue + } + + if !e.Config.Passive { + e.processMetrics(services) + psec := e.DNSQueriesPerSec() + // Check if it's too soon to release the next domain name + if psec > 0 && ((e.DNSNamesRemaining()*len(InitialQueryTypes))/psec) > tickSeconds { + continue + } } + // Check if the next domain should be sent to data sources/brute forcing + domainIdx++ + e.releaseDomainName(domainIdx) } } t.Stop() @@ -209,6 +231,25 @@ loop: return nil } +func (e *Enumeration) releaseDomainName(idx int) { + domains := e.Config.Domains() + + if idx >= len(domains) { + return + } + + for _, srv := range append(e.dataSources, e.bruteSrv) { + if srv == nil { + continue + } + + srv.SendRequest(&core.Request{ + Name: domains[idx], + Domain: domains[idx], + }) + } +} + func (e *Enumeration) submitKnownNames() { for _, enum := range e.Graph.EnumerationList() { var found bool @@ -253,10 +294,6 @@ func (e *Enumeration) DNSNamesRemaining() int { } func (e *Enumeration) processMetrics(services []core.Service) { - if e.Config.Passive { - return - } - var total, remaining int for _, srv := range services { stats := srv.Stats() @@ -265,8 +302,6 @@ func (e *Enumeration) processMetrics(services []core.Service) { total += stats.DNSQueriesPerSec } - e.Config.Log.Printf("Average DNS queries performed: %d/sec, DNS names remaining: %d", total, remaining) - e.metricsLock.Lock() e.dnsQueriesPerSec = total e.dnsNamesRemaining = remaining diff --git a/amass/amass_test.go b/amass/amass_test.go new file mode 100644 index 000000000..06194388d --- /dev/null +++ b/amass/amass_test.go @@ -0,0 +1,23 @@ +// Copyright 2017 Jeff Foley. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package amass + +import ( + "flag" + "os" + "testing" +) + +var ( + network = flag.Bool("network", false, "Run tests that require connectivity (take more time)") +) + +// TestMain will parse the test flags and setup for integration tests. +func TestMain(m *testing.M) { + flag.Parse() + + result := m.Run() + + os.Exit(result) +} diff --git a/amass/brute.go b/amass/brute.go index 53c89ed76..2da9abe34 100644 --- a/amass/brute.go +++ b/amass/brute.go @@ -36,7 +36,7 @@ type BruteForceService struct { // NewBruteForceService returns he object initialized, but not yet started. func NewBruteForceService(config *core.Config, bus *core.EventBus) *BruteForceService { bfs := &BruteForceService{ - max: utils.NewSimpleSemaphore(5000), + max: utils.NewSimpleSemaphore(50000), filter: utils.NewStringFilter(), } @@ -87,12 +87,15 @@ func (bfs *BruteForceService) processRequests() { } func (bfs *BruteForceService) goodRequest(req *core.Request) bool { - var ok bool if !bfs.Config().IsDomainInScope(req.Name) { - return ok + return false + } + + if len(req.Records) == 0 { + return true } - bfs.SetActive() + var ok bool for _, r := range req.Records { t := uint16(r.Type) @@ -178,6 +181,9 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) { bfs.metrics.QueryTime(time.Now()) bfs.SetActive() } + if len(answers) == 0 { + return + } req := &core.Request{ Name: name, @@ -187,8 +193,7 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) { Source: bfs.String(), } - bfs.SetActive() - if len(answers) == 0 || MatchesWildcard(req) { + if MatchesWildcard(req) { return } bfs.Bus().Publish(core.NameResolvedTopic, req) diff --git a/amass/core/request.go b/amass/core/request.go index ff9ed07b9..5e8bf24da 100644 --- a/amass/core/request.go +++ b/amass/core/request.go @@ -66,10 +66,11 @@ type Output struct { // AddressInfo stores all network addressing info for the Output type. type AddressInfo struct { - Address net.IP `json:"ip"` - Netblock *net.IPNet `json:"cidr"` - ASN int `json:"asn"` - Description string `json:"desc"` + Address net.IP `json:"ip"` + Netblock *net.IPNet + CIDRStr string `json:"cidr"` + ASN int `json:"asn"` + Description string `json:"desc"` } type pubReq struct { diff --git a/amass/dnssrv.go b/amass/dnssrv.go index b24725b07..0b2533e20 100644 --- a/amass/dnssrv.go +++ b/amass/dnssrv.go @@ -48,7 +48,7 @@ type DNSService struct { // NewDNSService returns he object initialized, but not yet started. func NewDNSService(config *core.Config, bus *core.EventBus) *DNSService { ds := &DNSService{ - max: utils.NewSimpleSemaphore(5000), + max: utils.NewSimpleSemaphore(50000), filter: utils.NewStringFilter(), } @@ -73,10 +73,6 @@ func (ds *DNSService) OnStart() error { ds.Bus().Subscribe(core.ReverseSweepTopic, ds.dnsSweep) ds.Bus().Subscribe(core.NewSubdomainTopic, ds.newSubdomain) go ds.processRequests() - - for _, domain := range ds.Config().Domains() { - go ds.basicQueries(domain, domain) - } return nil } diff --git a/amass/handlers/graph.go b/amass/handlers/graph.go index b677784a5..0c9774ee1 100644 --- a/amass/handlers/graph.go +++ b/amass/handlers/graph.go @@ -699,6 +699,7 @@ func (g *Graph) buildAddrInfo(addr, uuid string) *core.AddressInfo { if cidr == "" { return nil } + ainfo.CIDRStr = cidr _, ainfo.Netblock, _ = net.ParseCIDR(cidr) p := cayley.StartPath(g.store, quad.String(cidr)).LabelContext(u).In(quad.String("has_prefix")) diff --git a/amass/handlers/gremlin.go b/amass/handlers/gremlin.go index e81da0064..6c758ed6d 100644 --- a/amass/handlers/gremlin.go +++ b/amass/handlers/gremlin.go @@ -785,6 +785,7 @@ func dataToOutput(path []*DataOptsParams) []*core.Output { case "address": addrinfo.Address = net.ParseIP(v.Address) case "netblock": + addrinfo.CIDRStr = v.CIDR _, addrinfo.Netblock, _ = net.ParseCIDR(v.CIDR) case "as": addrinfo.ASN = v.ASN diff --git a/amass/resolvers.go b/amass/resolvers.go index 7c4265ea9..16f2c5057 100644 --- a/amass/resolvers.go +++ b/amass/resolvers.go @@ -36,6 +36,7 @@ var ( retryCodes = []int{ dns.RcodeRefused, + dns.RcodeServerFailure, dns.RcodeNotImplemented, } @@ -48,6 +49,15 @@ func init() { } } +type resolveError struct { + Err string + Rcode int +} + +func (e *resolveError) Error() string { + return e.Err +} + type resolveRequest struct { Timestamp time.Time Name string @@ -55,12 +65,27 @@ type resolveRequest struct { Result chan *resolveResult } +func (r *resolver) returnRequest(req *resolveRequest, res *resolveResult) { + req.Result <- res +} + type resolveResult struct { Records []core.DNSAnswer Again bool Err error } +func makeResolveResult(rec []core.DNSAnswer, again bool, err string, rcode int) *resolveResult { + return &resolveResult{ + Records: rec, + Again: again, + Err: &resolveError{ + Err: err, + Rcode: rcode, + }, + } +} + type resolver struct { sync.RWMutex Address string @@ -164,11 +189,8 @@ func (r *resolver) checkForTimeouts() { // Remove the timed out requests from the map for _, id := range timeouts { if req := r.pullRequest(id); req != nil { - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: true, - Err: fmt.Errorf("DNS query for %s, type %d timed out", req.Name, req.Qtype), - }) + estr := fmt.Sprintf("DNS query for %s, type %d timed out", req.Name, req.Qtype) + r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) } } // Complete handling of the timed out requests @@ -188,10 +210,6 @@ func (r *resolver) resolve(name string, qtype uint16) ([]core.DNSAnswer, bool, e return result.Records, result.Again, result.Err } -func (r *resolver) returnRequest(req *resolveRequest, res *resolveResult) { - req.Result <- res -} - func (r *resolver) fillXchgChan() { curIdx := 0 maxIdx := 7 @@ -239,11 +257,8 @@ func (r *resolver) writeMessage(co *dns.Conn, req *resolveRequest) { co.SetWriteDeadline(time.Now().Add(r.WindowDuration)) if err := co.WriteMsg(msg); err != nil { r.pullRequest(msg.MsgHdr.Id) - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: true, - Err: fmt.Errorf("DNS error: Failed to write query msg: %v", err), - }) + estr := fmt.Sprintf("DNS error: Failed to write query msg: %v", err) + r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) return } @@ -272,11 +287,8 @@ func (r *resolver) tcpExchange(req *resolveRequest) { conn, err := d.Dial("tcp", r.Address) if err != nil { r.pullRequest(msg.MsgHdr.Id) - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: true, - Err: fmt.Errorf("DNS: Failed to obtain TCP connection to %s: %v", r.Address, err), - }) + estr := fmt.Sprintf("DNS: Failed to obtain TCP connection to %s: %v", r.Address, err) + r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) return } defer conn.Close() @@ -285,11 +297,8 @@ func (r *resolver) tcpExchange(req *resolveRequest) { co.SetWriteDeadline(time.Now().Add(r.WindowDuration)) if err := co.WriteMsg(msg); err != nil { r.pullRequest(msg.MsgHdr.Id) - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: true, - Err: fmt.Errorf("DNS error: Failed to write query msg: %v", err), - }) + estr := fmt.Sprintf("DNS error: Failed to write query msg: %v", err) + r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) return } @@ -299,11 +308,8 @@ func (r *resolver) tcpExchange(req *resolveRequest) { read, err := co.ReadMsg() if read == nil || err != nil { r.pullRequest(msg.MsgHdr.Id) - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: true, - Err: fmt.Errorf("DNS error: Failed to read the reply msg: %v", err), - }) + estr := fmt.Sprintf("DNS error: Failed to read the reply msg: %v", err) + r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) return } @@ -325,12 +331,9 @@ func (r *resolver) processMessage(msg *dns.Msg) { break } } - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: again, - Err: fmt.Errorf("DNS query for %s, type %d returned error %s", - req.Name, req.Qtype, dns.RcodeToString[msg.Rcode]), - }) + estr := fmt.Sprintf("DNS query for %s, type %d returned error %s", + req.Name, req.Qtype, dns.RcodeToString[msg.Rcode]) + r.returnRequest(req, makeResolveResult(nil, again, estr, msg.Rcode)) return } @@ -350,11 +353,8 @@ func (r *resolver) processMessage(msg *dns.Msg) { } if len(answers) == 0 { - r.returnRequest(req, &resolveResult{ - Records: nil, - Again: false, - Err: fmt.Errorf("DNS query for %s, type %d returned 0 records", req.Name, req.Qtype), - }) + estr := fmt.Sprintf("DNS query for %s, type %d returned 0 records", req.Name, req.Qtype) + r.returnRequest(req, makeResolveResult(nil, false, estr, msg.Rcode)) return } @@ -440,7 +440,7 @@ func (r *resolver) Available() bool { r.Unlock() // There needs to be an opportunity to exceed the success rate if !avail { - if random := randomInt(1, 100); random <= 10 { + if random := randomInt(1, 100); random <= 5 { avail = true } } @@ -503,23 +503,35 @@ func SetCustomResolvers(res []string) { func Resolve(name, qtype string) ([]core.DNSAnswer, error) { qt, err := textToTypeNum(qtype) if err != nil { - return nil, err + return nil, &resolveError{ + Err: err.Error(), + Rcode: 100, + } } var again bool - var attempts int + start := time.Now() var ans []core.DNSAnswer - cutoff := time.Now().Add(30 * time.Second) + var attempts, servfail int for { ans, again, err = nextResolver().resolve(name, qt) if !again { break } - // Do not allow retries to continue forever + attempts++ - if attempts > maxRetries && time.Now().After(cutoff) { + if attempts > 50 && time.Now().After(start.Add(2*time.Minute)) { break } + // Do not allow server failure errors to continue as long + if (err.(*resolveError)).Rcode == dns.RcodeServerFailure { + servfail++ + if servfail > 10 && time.Now().After(start.Add(time.Minute)) { + break + } else if servfail <= 5 { + time.Sleep(time.Duration(randomInt(3000, 5000)) * time.Millisecond) + } + } } return ans, err } @@ -534,7 +546,10 @@ func Reverse(addr string) (string, string, error) { } else if len(ip) == net.IPv6len { ptr = utils.IPv6NibbleFormat(utils.HexString(ip)) + ".ip6.arpa" } else { - return ptr, "", fmt.Errorf("Invalid IP address parameter: %s", addr) + return ptr, "", &resolveError{ + Err: fmt.Sprintf("Invalid IP address parameter: %s", addr), + Rcode: 100, + } } answers, err := Resolve(ptr, "PTR") @@ -550,9 +565,15 @@ func Reverse(addr string) (string, string, error) { } if name == "" { - err = fmt.Errorf("PTR record not found for IP address: %s", addr) + err = &resolveError{ + Err: fmt.Sprintf("PTR record not found for IP address: %s", addr), + Rcode: 100, + } } else if strings.HasSuffix(name, ".in-addr.arpa") || strings.HasSuffix(name, ".ip6.arpa") { - err = fmt.Errorf("Invalid target in PTR record answer: %s", name) + err = &resolveError{ + Err: fmt.Sprintf("Invalid target in PTR record answer: %s", name), + Rcode: 100, + } } return ptr, name, err } @@ -768,58 +789,58 @@ func extractRawData(msg *dns.Msg, qtype uint16) []string { for _, a := range msg.Answer { if a.Header().Rrtype == qtype { + var value string + switch qtype { case dns.TypeA: if t, ok := a.(*dns.A); ok { - data = append(data, utils.CopyString(t.A.String())) + value = utils.CopyString(t.A.String()) } case dns.TypeAAAA: if t, ok := a.(*dns.AAAA); ok { - data = append(data, utils.CopyString(t.AAAA.String())) + value = utils.CopyString(t.AAAA.String()) } case dns.TypeCNAME: if t, ok := a.(*dns.CNAME); ok { - data = append(data, utils.CopyString(t.Target)) + value = utils.CopyString(t.Target) } case dns.TypePTR: if t, ok := a.(*dns.PTR); ok { - data = append(data, utils.CopyString(t.Ptr)) + value = utils.CopyString(t.Ptr) } case dns.TypeNS: if t, ok := a.(*dns.NS); ok { - data = append(data, realName(t.Hdr)+","+removeLastDot(t.Ns)) + value = realName(t.Hdr) + "," + removeLastDot(t.Ns) } case dns.TypeMX: if t, ok := a.(*dns.MX); ok { - data = append(data, utils.CopyString(t.Mx)) + value = utils.CopyString(t.Mx) } case dns.TypeTXT: if t, ok := a.(*dns.TXT); ok { - var all string - for _, piece := range t.Txt { - all += piece + " " + value += piece + " " } - data = append(data, all) } case dns.TypeSOA: if t, ok := a.(*dns.SOA); ok { - data = append(data, t.Ns+" "+t.Mbox) + value = t.Ns + " " + t.Mbox } case dns.TypeSPF: if t, ok := a.(*dns.SPF); ok { - var all string - for _, piece := range t.Txt { - all += piece + " " + value += piece + " " } - data = append(data, all) } case dns.TypeSRV: if t, ok := a.(*dns.SRV); ok { - data = append(data, utils.CopyString(t.Target)) + value = utils.CopyString(t.Target) } } + + if value != "" { + data = append(data, strings.TrimSpace(value)) + } } } return data diff --git a/amass/sources/archiveit.go b/amass/sources/archiveit.go index 6cce875d6..f7703e8aa 100644 --- a/amass/sources/archiveit.go +++ b/amass/sources/archiveit.go @@ -34,18 +34,10 @@ func (a *ArchiveIt) OnStart() error { a.BaseService.OnStart() a.Bus().Subscribe(core.NameResolvedTopic, a.SendRequest) - go a.startRootDomains() go a.processRequests() return nil } -func (a *ArchiveIt) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range a.Config().Domains() { - a.executeQuery(domain, domain) - } -} - func (a *ArchiveIt) processRequests() { for { select { diff --git a/amass/sources/archivetoday.go b/amass/sources/archivetoday.go index dfdeba28e..684b096ba 100644 --- a/amass/sources/archivetoday.go +++ b/amass/sources/archivetoday.go @@ -34,18 +34,10 @@ func (a *ArchiveToday) OnStart() error { a.BaseService.OnStart() a.Bus().Subscribe(core.NameResolvedTopic, a.SendRequest) - go a.startRootDomains() go a.processRequests() return nil } -func (a *ArchiveToday) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range a.Config().Domains() { - a.executeQuery(domain, domain) - } -} - func (a *ArchiveToday) processRequests() { for { select { diff --git a/amass/sources/arquivo.go b/amass/sources/arquivo.go index da9022aca..857d5f62e 100644 --- a/amass/sources/arquivo.go +++ b/amass/sources/arquivo.go @@ -34,18 +34,10 @@ func (a *Arquivo) OnStart() error { a.BaseService.OnStart() a.Bus().Subscribe(core.NameResolvedTopic, a.SendRequest) - go a.startRootDomains() go a.processRequests() return nil } -func (a *Arquivo) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range a.Config().Domains() { - a.executeQuery(domain, domain) - } -} - func (a *Arquivo) processRequests() { for { select { diff --git a/amass/sources/ask.go b/amass/sources/ask.go index 3cb34f804..1f2165ca0 100644 --- a/amass/sources/ask.go +++ b/amass/sources/ask.go @@ -37,14 +37,20 @@ func NewAsk(config *core.Config, bus *core.EventBus) *Ask { func (a *Ask) OnStart() error { a.BaseService.OnStart() - go a.startRootDomains() + go a.processRequests() return nil } -func (a *Ask) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range a.Config().Domains() { - a.executeQuery(domain) +func (a *Ask) processRequests() { + for { + select { + case <-a.Quit(): + return + case req := <-a.RequestChan(): + if a.Config().IsDomainInScope(req.Domain) { + a.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/baidu.go b/amass/sources/baidu.go index b342fe17d..7aa986bd1 100644 --- a/amass/sources/baidu.go +++ b/amass/sources/baidu.go @@ -37,14 +37,20 @@ func NewBaidu(config *core.Config, bus *core.EventBus) *Baidu { func (b *Baidu) OnStart() error { b.BaseService.OnStart() - go b.startRootDomains() + go b.processRequests() return nil } -func (b *Baidu) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range b.Config().Domains() { - b.executeQuery(domain) +func (b *Baidu) processRequests() { + for { + select { + case <-b.Quit(): + return + case req := <-b.RequestChan(): + if b.Config().IsDomainInScope(req.Domain) { + b.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/binaryedge.go b/amass/sources/binaryedge.go index f63a9d503..45cefacc1 100644 --- a/amass/sources/binaryedge.go +++ b/amass/sources/binaryedge.go @@ -39,16 +39,28 @@ func (be *BinaryEdge) OnStart() error { if be.API == nil || be.API.Key == "" { be.Config().Log.Printf("%s: API key data was not provided", be.String()) } - go be.startRootDomains() + + go be.processRequests() return nil } -func (be *BinaryEdge) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range be.Config().Domains() { - be.executeQuery(domain) - // Honor the rate limit - time.Sleep(be.RateLimit) +func (be *BinaryEdge) processRequests() { + last := time.Now() + + for { + select { + case <-be.Quit(): + return + case req := <-be.RequestChan(): + if be.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < be.RateLimit { + time.Sleep(be.RateLimit) + } + + be.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/bing.go b/amass/sources/bing.go index c75a5f956..40cb4cb84 100644 --- a/amass/sources/bing.go +++ b/amass/sources/bing.go @@ -37,14 +37,20 @@ func NewBing(config *core.Config, bus *core.EventBus) *Bing { func (b *Bing) OnStart() error { b.BaseService.OnStart() - go b.startRootDomains() + go b.processRequests() return nil } -func (b *Bing) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range b.Config().Domains() { - b.executeQuery(domain) +func (b *Bing) processRequests() { + for { + select { + case <-b.Quit(): + return + case req := <-b.RequestChan(): + if b.Config().IsDomainInScope(req.Domain) { + b.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/bufferover.go b/amass/sources/bufferover.go index fb7f711db..9c210321f 100644 --- a/amass/sources/bufferover.go +++ b/amass/sources/bufferover.go @@ -29,14 +29,20 @@ func NewBufferOver(config *core.Config, bus *core.EventBus) *BufferOver { func (b *BufferOver) OnStart() error { b.BaseService.OnStart() - go b.startRootDomains() + go b.processRequests() return nil } -func (b *BufferOver) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range b.Config().Domains() { - b.executeQuery(domain) +func (b *BufferOver) processRequests() { + for { + select { + case <-b.Quit(): + return + case req := <-b.RequestChan(): + if b.Config().IsDomainInScope(req.Domain) { + b.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/censys.go b/amass/sources/censys.go index 7bd3b826d..6f0468ef9 100644 --- a/amass/sources/censys.go +++ b/amass/sources/censys.go @@ -41,16 +41,28 @@ func (c *Censys) OnStart() error { if c.API == nil || c.API.Key == "" || c.API.Secret == "" { c.Config().Log.Printf("%s: API key data was not provided", c.String()) } - go c.startRootDomains() + + go c.processRequests() return nil } -func (c *Censys) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range c.Config().Domains() { - c.executeQuery(domain) - // Honor the rate limit - time.Sleep(c.RateLimit) +func (c *Censys) processRequests() { + last := time.Now() + + for { + select { + case <-c.Quit(): + return + case req := <-c.RequestChan(): + if c.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < c.RateLimit { + time.Sleep(c.RateLimit) + } + + c.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/certdb.go b/amass/sources/certdb.go index 4f8058aea..96cefc245 100644 --- a/amass/sources/certdb.go +++ b/amass/sources/certdb.go @@ -40,14 +40,20 @@ func (c *CertDB) OnStart() error { c.authenticate() } - go c.startRootDomains() + go c.processRequests() return nil } -func (c *CertDB) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range c.Config().Domains() { - c.executeQuery(domain) +func (c *CertDB) processRequests() { + for { + select { + case <-c.Quit(): + return + case req := <-c.RequestChan(): + if c.Config().IsDomainInScope(req.Domain) { + c.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/certspotter.go b/amass/sources/certspotter.go index 6b0532c7c..2308c20bf 100644 --- a/amass/sources/certspotter.go +++ b/amass/sources/certspotter.go @@ -29,14 +29,20 @@ func NewCertSpotter(config *core.Config, bus *core.EventBus) *CertSpotter { func (c *CertSpotter) OnStart() error { c.BaseService.OnStart() - go c.startRootDomains() + go c.processRequests() return nil } -func (c *CertSpotter) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range c.Config().Domains() { - c.executeQuery(domain) +func (c *CertSpotter) processRequests() { + for { + select { + case <-c.Quit(): + return + case req := <-c.RequestChan(): + if c.Config().IsDomainInScope(req.Domain) { + c.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/circl.go b/amass/sources/circl.go index ee0b2d4d7..2bc808aa1 100644 --- a/amass/sources/circl.go +++ b/amass/sources/circl.go @@ -41,16 +41,28 @@ func (c *CIRCL) OnStart() error { if c.API == nil || c.API.Username == "" || c.API.Password == "" { c.Config().Log.Printf("%s: API key data was not provided", c.String()) } - go c.startRootDomains() + + go c.processRequests() return nil } -func (c *CIRCL) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range c.Config().Domains() { - c.executeQuery(domain) - // Honor the rate limit - time.Sleep(c.RateLimit) +func (c *CIRCL) processRequests() { + last := time.Now() + + for { + select { + case <-c.Quit(): + return + case req := <-c.RequestChan(): + if c.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < c.RateLimit { + time.Sleep(c.RateLimit) + } + + c.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/commoncrawl.go b/amass/sources/commoncrawl.go index 5a8db821b..94596896c 100644 --- a/amass/sources/commoncrawl.go +++ b/amass/sources/commoncrawl.go @@ -51,14 +51,20 @@ func NewCommonCrawl(config *core.Config, bus *core.EventBus) *CommonCrawl { func (c *CommonCrawl) OnStart() error { c.BaseService.OnStart() - go c.startRootDomains() + go c.processRequests() return nil } -func (c *CommonCrawl) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range c.Config().Domains() { - c.executeQuery(domain) +func (c *CommonCrawl) processRequests() { + for { + select { + case <-c.Quit(): + return + case req := <-c.RequestChan(): + if c.Config().IsDomainInScope(req.Domain) { + c.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/crtsh.go b/amass/sources/crtsh.go index e6d6bb6ea..8e262211b 100644 --- a/amass/sources/crtsh.go +++ b/amass/sources/crtsh.go @@ -29,14 +29,20 @@ func NewCrtsh(config *core.Config, bus *core.EventBus) *Crtsh { func (c *Crtsh) OnStart() error { c.BaseService.OnStart() - go c.startRootDomains() + go c.processRequests() return nil } -func (c *Crtsh) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range c.Config().Domains() { - c.executeQuery(domain) +func (c *Crtsh) processRequests() { + for { + select { + case <-c.Quit(): + return + case req := <-c.RequestChan(): + if c.Config().IsDomainInScope(req.Domain) { + c.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/dnsdb.go b/amass/sources/dnsdb.go index 6a3749b35..29d5bd6c3 100644 --- a/amass/sources/dnsdb.go +++ b/amass/sources/dnsdb.go @@ -43,16 +43,28 @@ func (d *DNSDB) OnStart() error { if d.API == nil || d.API.Key == "" { d.Config().Log.Printf("%s: API key data was not provided", d.String()) } - go d.startRootDomains() + + go d.processRequests() return nil } -func (d *DNSDB) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range d.Config().Domains() { - d.executeQuery(domain) - // Honor the rate limit - time.Sleep(d.RateLimit) +func (d *DNSDB) processRequests() { + last := time.Now() + + for { + select { + case <-d.Quit(): + return + case req := <-d.RequestChan(): + if d.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < d.RateLimit { + time.Sleep(d.RateLimit) + } + + d.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/dnsdumpster.go b/amass/sources/dnsdumpster.go index 5fea71d23..5b787bd31 100644 --- a/amass/sources/dnsdumpster.go +++ b/amass/sources/dnsdumpster.go @@ -35,14 +35,20 @@ func NewDNSDumpster(config *core.Config, bus *core.EventBus) *DNSDumpster { func (d *DNSDumpster) OnStart() error { d.BaseService.OnStart() - go d.startRootDomains() + go d.processRequests() return nil } -func (d *DNSDumpster) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range d.Config().Domains() { - d.executeQuery(domain) +func (d *DNSDumpster) processRequests() { + for { + select { + case <-d.Quit(): + return + case req := <-d.RequestChan(): + if d.Config().IsDomainInScope(req.Domain) { + d.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/dnstable.go b/amass/sources/dnstable.go index f108cb4b5..c6afeeac5 100644 --- a/amass/sources/dnstable.go +++ b/amass/sources/dnstable.go @@ -29,14 +29,20 @@ func NewDNSTable(config *core.Config, bus *core.EventBus) *DNSTable { func (d *DNSTable) OnStart() error { d.BaseService.OnStart() - go d.startRootDomains() + go d.processRequests() return nil } -func (d *DNSTable) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range d.Config().Domains() { - d.executeQuery(domain) +func (d *DNSTable) processRequests() { + for { + select { + case <-d.Quit(): + return + case req := <-d.RequestChan(): + if d.Config().IsDomainInScope(req.Domain) { + d.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/dogpile.go b/amass/sources/dogpile.go index 83ee55fbc..464f219b6 100644 --- a/amass/sources/dogpile.go +++ b/amass/sources/dogpile.go @@ -37,14 +37,20 @@ func NewDogpile(config *core.Config, bus *core.EventBus) *Dogpile { func (d *Dogpile) OnStart() error { d.BaseService.OnStart() - go d.startRootDomains() + go d.processRequests() return nil } -func (d *Dogpile) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range d.Config().Domains() { - d.executeQuery(domain) +func (d *Dogpile) processRequests() { + for { + select { + case <-d.Quit(): + return + case req := <-d.RequestChan(): + if d.Config().IsDomainInScope(req.Domain) { + d.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/entrust.go b/amass/sources/entrust.go index f1caccc95..a395077f5 100644 --- a/amass/sources/entrust.go +++ b/amass/sources/entrust.go @@ -31,14 +31,20 @@ func NewEntrust(config *core.Config, bus *core.EventBus) *Entrust { func (e *Entrust) OnStart() error { e.BaseService.OnStart() - go e.startRootDomains() + go e.processRequests() return nil } -func (e *Entrust) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range e.Config().Domains() { - e.executeQuery(domain) +func (e *Entrust) processRequests() { + for { + select { + case <-e.Quit(): + return + case req := <-e.RequestChan(): + if e.Config().IsDomainInScope(req.Domain) { + e.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/exalead.go b/amass/sources/exalead.go index 80750f77d..e1e483995 100644 --- a/amass/sources/exalead.go +++ b/amass/sources/exalead.go @@ -29,14 +29,20 @@ func NewExalead(config *core.Config, bus *core.EventBus) *Exalead { func (e *Exalead) OnStart() error { e.BaseService.OnStart() - go e.startRootDomains() + go e.processRequests() return nil } -func (e *Exalead) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range e.Config().Domains() { - e.executeQuery(domain) +func (e *Exalead) processRequests() { + for { + select { + case <-e.Quit(): + return + case req := <-e.RequestChan(): + if e.Config().IsDomainInScope(req.Domain) { + e.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/findsubdomains.go b/amass/sources/findsubdomains.go index 8a2a15e56..e20adf88f 100644 --- a/amass/sources/findsubdomains.go +++ b/amass/sources/findsubdomains.go @@ -29,14 +29,20 @@ func NewFindSubdomains(config *core.Config, bus *core.EventBus) *FindSubdomains func (f *FindSubdomains) OnStart() error { f.BaseService.OnStart() - go f.startRootDomains() + go f.processRequests() return nil } -func (f *FindSubdomains) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range f.Config().Domains() { - f.executeQuery(domain) +func (f *FindSubdomains) processRequests() { + for { + select { + case <-f.Quit(): + return + case req := <-f.RequestChan(): + if f.Config().IsDomainInScope(req.Domain) { + f.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/google.go b/amass/sources/google.go index 3191fd57f..6f63496e9 100644 --- a/amass/sources/google.go +++ b/amass/sources/google.go @@ -37,14 +37,20 @@ func NewGoogle(config *core.Config, bus *core.EventBus) *Google { func (g *Google) OnStart() error { g.BaseService.OnStart() - go g.startRootDomains() + go g.processRequests() return nil } -func (g *Google) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range g.Config().Domains() { - g.executeQuery(domain) +func (g *Google) processRequests() { + for { + select { + case <-g.Quit(): + return + case req := <-g.RequestChan(): + if g.Config().IsDomainInScope(req.Domain) { + g.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/hackertarget.go b/amass/sources/hackertarget.go index b0964ae0b..ba0d1b91b 100644 --- a/amass/sources/hackertarget.go +++ b/amass/sources/hackertarget.go @@ -29,14 +29,20 @@ func NewHackerTarget(config *core.Config, bus *core.EventBus) *HackerTarget { func (h *HackerTarget) OnStart() error { h.BaseService.OnStart() - go h.startRootDomains() + go h.processRequests() return nil } -func (h *HackerTarget) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range h.Config().Domains() { - h.executeQuery(domain) +func (h *HackerTarget) processRequests() { + for { + select { + case <-h.Quit(): + return + case req := <-h.RequestChan(): + if h.Config().IsDomainInScope(req.Domain) { + h.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/ipv4info.go b/amass/sources/ipv4info.go index c610708df..d24c13c63 100644 --- a/amass/sources/ipv4info.go +++ b/amass/sources/ipv4info.go @@ -35,14 +35,20 @@ func NewIPv4Info(config *core.Config, bus *core.EventBus) *IPv4Info { func (i *IPv4Info) OnStart() error { i.BaseService.OnStart() - go i.startRootDomains() + go i.processRequests() return nil } -func (i *IPv4Info) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range i.Config().Domains() { - i.executeQuery(domain) +func (i *IPv4Info) processRequests() { + for { + select { + case <-i.Quit(): + return + case req := <-i.RequestChan(): + if i.Config().IsDomainInScope(req.Domain) { + i.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/locarchive.go b/amass/sources/locarchive.go index f375afaf0..ea943f23a 100644 --- a/amass/sources/locarchive.go +++ b/amass/sources/locarchive.go @@ -33,18 +33,11 @@ func NewLoCArchive(config *core.Config, bus *core.EventBus) *LoCArchive { func (l *LoCArchive) OnStart() error { l.BaseService.OnStart() - go l.startRootDomains() + l.Bus().Subscribe(core.NameResolvedTopic, l.SendRequest) go l.processRequests() return nil } -func (l *LoCArchive) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range l.Config().Domains() { - l.executeQuery(domain, domain) - } -} - func (l *LoCArchive) processRequests() { for { select { diff --git a/amass/sources/netcraft.go b/amass/sources/netcraft.go index eaf958611..fbcf555cb 100644 --- a/amass/sources/netcraft.go +++ b/amass/sources/netcraft.go @@ -29,14 +29,20 @@ func NewNetcraft(config *core.Config, bus *core.EventBus) *Netcraft { func (n *Netcraft) OnStart() error { n.BaseService.OnStart() - go n.startRootDomains() + go n.processRequests() return nil } -func (n *Netcraft) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range n.Config().Domains() { - n.executeQuery(domain) +func (n *Netcraft) processRequests() { + for { + select { + case <-n.Quit(): + return + case req := <-n.RequestChan(): + if n.Config().IsDomainInScope(req.Domain) { + n.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/openukarchive.go b/amass/sources/openukarchive.go index 03efef04e..b74c003b3 100644 --- a/amass/sources/openukarchive.go +++ b/amass/sources/openukarchive.go @@ -34,18 +34,10 @@ func (o *OpenUKArchive) OnStart() error { o.BaseService.OnStart() o.Bus().Subscribe(core.NameResolvedTopic, o.SendRequest) - go o.startRootDomains() go o.processRequests() return nil } -func (o *OpenUKArchive) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range o.Config().Domains() { - o.executeQuery(domain, domain) - } -} - func (o *OpenUKArchive) processRequests() { for { select { diff --git a/amass/sources/passivetotal.go b/amass/sources/passivetotal.go index dac0561f1..07a93b891 100644 --- a/amass/sources/passivetotal.go +++ b/amass/sources/passivetotal.go @@ -39,16 +39,28 @@ func (pt *PassiveTotal) OnStart() error { if pt.API == nil || pt.API.Username == "" || pt.API.Key == "" { pt.Config().Log.Printf("%s: API key data was not provided", pt.String()) } - go pt.startRootDomains() + + go pt.processRequests() return nil } -func (pt *PassiveTotal) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range pt.Config().Domains() { - pt.executeQuery(domain) - // Honor the rate limit - time.Sleep(pt.RateLimit) +func (pt *PassiveTotal) processRequests() { + last := time.Now() + + for { + select { + case <-pt.Quit(): + return + case req := <-pt.RequestChan(): + if pt.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < pt.RateLimit { + time.Sleep(pt.RateLimit) + } + + pt.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/ptrarchive.go b/amass/sources/ptrarchive.go index 44f9df21b..e6f5f2c01 100644 --- a/amass/sources/ptrarchive.go +++ b/amass/sources/ptrarchive.go @@ -29,14 +29,20 @@ func NewPTRArchive(config *core.Config, bus *core.EventBus) *PTRArchive { func (p *PTRArchive) OnStart() error { p.BaseService.OnStart() - go p.startRootDomains() + go p.processRequests() return nil } -func (p *PTRArchive) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range p.Config().Domains() { - p.executeQuery(domain) +func (p *PTRArchive) processRequests() { + for { + select { + case <-p.Quit(): + return + case req := <-p.RequestChan(): + if p.Config().IsDomainInScope(req.Domain) { + p.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/riddler.go b/amass/sources/riddler.go index aa12d2902..1a1d08fb1 100644 --- a/amass/sources/riddler.go +++ b/amass/sources/riddler.go @@ -29,14 +29,20 @@ func NewRiddler(config *core.Config, bus *core.EventBus) *Riddler { func (r *Riddler) OnStart() error { r.BaseService.OnStart() - go r.startRootDomains() + go r.processRequests() return nil } -func (r *Riddler) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range r.Config().Domains() { - r.executeQuery(domain) +func (r *Riddler) processRequests() { + for { + select { + case <-r.Quit(): + return + case req := <-r.RequestChan(): + if r.Config().IsDomainInScope(req.Domain) { + r.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/robtex.go b/amass/sources/robtex.go index c99b6d468..c2e597bac 100644 --- a/amass/sources/robtex.go +++ b/amass/sources/robtex.go @@ -38,14 +38,20 @@ func NewRobtex(config *core.Config, bus *core.EventBus) *Robtex { func (r *Robtex) OnStart() error { r.BaseService.OnStart() - go r.startRootDomains() + go r.processRequests() return nil } -func (r *Robtex) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range r.Config().Domains() { - r.executeQuery(domain) +func (r *Robtex) processRequests() { + for { + select { + case <-r.Quit(): + return + case req := <-r.RequestChan(): + if r.Config().IsDomainInScope(req.Domain) { + r.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/securitytrails.go b/amass/sources/securitytrails.go index 2fd710061..2cb3c034d 100644 --- a/amass/sources/securitytrails.go +++ b/amass/sources/securitytrails.go @@ -40,16 +40,28 @@ func (st *SecurityTrails) OnStart() error { if st.API == nil || st.API.Key == "" { st.Config().Log.Printf("%s: API key data was not provided", st.String()) } - go st.startRootDomains() + + go st.processRequests() return nil } -func (st *SecurityTrails) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range st.Config().Domains() { - st.executeQuery(domain) - // Honor the rate limit - time.Sleep(st.RateLimit) +func (st *SecurityTrails) processRequests() { + last := time.Now() + + for { + select { + case <-st.Quit(): + return + case req := <-st.RequestChan(): + if st.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < st.RateLimit { + time.Sleep(st.RateLimit) + } + + st.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/shodan.go b/amass/sources/shodan.go index 9eb6214fd..5efe55766 100644 --- a/amass/sources/shodan.go +++ b/amass/sources/shodan.go @@ -40,16 +40,28 @@ func (s *Shodan) OnStart() error { if s.API == nil || s.API.Key == "" { s.Config().Log.Printf("%s: API key data was not provided", s.String()) } - go s.startRootDomains() + + go s.processRequests() return nil } -func (s *Shodan) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range s.Config().Domains() { - s.executeQuery(domain) - // Honor the rate limit - time.Sleep(s.RateLimit) +func (s *Shodan) processRequests() { + last := time.Now() + + for { + select { + case <-s.Quit(): + return + case req := <-s.RequestChan(): + if s.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < s.RateLimit { + time.Sleep(s.RateLimit) + } + + s.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/sitedossier.go b/amass/sources/sitedossier.go index 9aa91c7e6..a7930974a 100644 --- a/amass/sources/sitedossier.go +++ b/amass/sources/sitedossier.go @@ -29,14 +29,20 @@ func NewSiteDossier(config *core.Config, bus *core.EventBus) *SiteDossier { func (s *SiteDossier) OnStart() error { s.BaseService.OnStart() - go s.startRootDomains() + go s.processRequests() return nil } -func (s *SiteDossier) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range s.Config().Domains() { - s.executeQuery(domain) +func (s *SiteDossier) processRequests() { + for { + select { + case <-s.Quit(): + return + case req := <-s.RequestChan(): + if s.Config().IsDomainInScope(req.Domain) { + s.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/sources_test.go b/amass/sources/sources_test.go index b22160c51..075bf279a 100644 --- a/amass/sources/sources_test.go +++ b/amass/sources/sources_test.go @@ -1,6 +1,26 @@ +// Copyright 2017 Jeff Foley. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + package sources -import "testing" +import ( + "flag" + "os" + "testing" +) + +var ( + network = flag.Bool("network", false, "Run tests that require connectivity (take more time)") +) + +// TestMain will parse the test flags and setup for integration tests. +func TestMain(m *testing.M) { + flag.Parse() + + result := m.Run() + + os.Exit(result) +} func TestCleanName(t *testing.T) { tests := []struct { diff --git a/amass/sources/threatcrowd.go b/amass/sources/threatcrowd.go index 4101c749e..f3f45fe19 100644 --- a/amass/sources/threatcrowd.go +++ b/amass/sources/threatcrowd.go @@ -29,14 +29,20 @@ func NewThreatCrowd(config *core.Config, bus *core.EventBus) *ThreatCrowd { func (t *ThreatCrowd) OnStart() error { t.BaseService.OnStart() - go t.startRootDomains() + go t.processRequests() return nil } -func (t *ThreatCrowd) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range t.Config().Domains() { - t.executeQuery(domain) +func (t *ThreatCrowd) processRequests() { + for { + select { + case <-t.Quit(): + return + case req := <-t.RequestChan(): + if t.Config().IsDomainInScope(req.Domain) { + t.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/twitter.go b/amass/sources/twitter.go index c416dbf57..3b772062d 100644 --- a/amass/sources/twitter.go +++ b/amass/sources/twitter.go @@ -54,16 +54,28 @@ func (t *Twitter) OnStart() error { t.client = twitter.NewClient(httpClient) } } - go t.startRootDomains() + + go t.processRequests() return nil } -func (t *Twitter) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range t.Config().Domains() { - t.executeQuery(domain) - // Honor the rate limit - time.Sleep(t.RateLimit) +func (t *Twitter) processRequests() { + last := time.Now() + + for { + select { + case <-t.Quit(): + return + case req := <-t.RequestChan(): + if t.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < t.RateLimit { + time.Sleep(t.RateLimit) + } + + t.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/ukgovarchive.go b/amass/sources/ukgovarchive.go index a86f7ab07..d54b38324 100644 --- a/amass/sources/ukgovarchive.go +++ b/amass/sources/ukgovarchive.go @@ -34,18 +34,10 @@ func (u *UKGovArchive) OnStart() error { u.BaseService.OnStart() u.Bus().Subscribe(core.NameResolvedTopic, u.SendRequest) - go u.startRootDomains() go u.processRequests() return nil } -func (u *UKGovArchive) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range u.Config().Domains() { - u.executeQuery(domain, domain) - } -} - func (u *UKGovArchive) processRequests() { for { select { diff --git a/amass/sources/umbrella.go b/amass/sources/umbrella.go index fc94d8eac..5382200bc 100644 --- a/amass/sources/umbrella.go +++ b/amass/sources/umbrella.go @@ -38,16 +38,28 @@ func (u *Umbrella) OnStart() error { if u.API == nil || u.API.Key == "" { u.Config().Log.Printf("%s: API key data was not provided", u.String()) } - go u.startRootDomains() + + go u.processRequests() return nil } -func (u *Umbrella) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range u.Config().Domains() { - u.executeQuery(domain) - // Honor the rate limit - time.Sleep(u.RateLimit) +func (u *Umbrella) processRequests() { + last := time.Now() + + for { + select { + case <-u.Quit(): + return + case req := <-u.RequestChan(): + if u.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < u.RateLimit { + time.Sleep(u.RateLimit) + } + + u.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/urlscan.go b/amass/sources/urlscan.go index 4ffc11518..d87f150d8 100644 --- a/amass/sources/urlscan.go +++ b/amass/sources/urlscan.go @@ -41,16 +41,28 @@ func (u *URLScan) OnStart() error { if u.API == nil || u.API.Key == "" { u.Config().Log.Printf("%s: API key data was not provided", u.String()) } - go u.startRootDomains() + + go u.processRequests() return nil } -func (u *URLScan) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range u.Config().Domains() { - u.executeQuery(domain) - // Honor the rate limit - time.Sleep(u.RateLimit) +func (u *URLScan) processRequests() { + last := time.Now() + + for { + select { + case <-u.Quit(): + return + case req := <-u.RequestChan(): + if u.Config().IsDomainInScope(req.Domain) { + if time.Now().Sub(last) < u.RateLimit { + time.Sleep(u.RateLimit) + } + + u.executeQuery(req.Domain) + last = time.Now() + } + } } } diff --git a/amass/sources/virustotal.go b/amass/sources/virustotal.go index 3e42470f9..aeedc8b62 100644 --- a/amass/sources/virustotal.go +++ b/amass/sources/virustotal.go @@ -29,14 +29,20 @@ func NewVirusTotal(config *core.Config, bus *core.EventBus) *VirusTotal { func (v *VirusTotal) OnStart() error { v.BaseService.OnStart() - go v.startRootDomains() + go v.processRequests() return nil } -func (v *VirusTotal) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range v.Config().Domains() { - v.executeQuery(domain) +func (v *VirusTotal) processRequests() { + for { + select { + case <-v.Quit(): + return + case req := <-v.RequestChan(): + if v.Config().IsDomainInScope(req.Domain) { + v.executeQuery(req.Domain) + } + } } } diff --git a/amass/sources/wayback.go b/amass/sources/wayback.go index 8bda2ac4d..762eebb73 100644 --- a/amass/sources/wayback.go +++ b/amass/sources/wayback.go @@ -34,18 +34,10 @@ func (w *Wayback) OnStart() error { w.BaseService.OnStart() w.Bus().Subscribe(core.NameResolvedTopic, w.SendRequest) - go w.startRootDomains() go w.processRequests() return nil } -func (w *Wayback) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range w.Config().Domains() { - w.executeQuery(domain, domain) - } -} - func (w *Wayback) processRequests() { for { select { diff --git a/amass/sources/yahoo.go b/amass/sources/yahoo.go index 3b8946fa4..0bbe00ae4 100644 --- a/amass/sources/yahoo.go +++ b/amass/sources/yahoo.go @@ -37,14 +37,20 @@ func NewYahoo(config *core.Config, bus *core.EventBus) *Yahoo { func (y *Yahoo) OnStart() error { y.BaseService.OnStart() - go y.startRootDomains() + go y.processRequests() return nil } -func (y *Yahoo) startRootDomains() { - // Look at each domain provided by the config - for _, domain := range y.Config().Domains() { - y.executeQuery(domain) +func (y *Yahoo) processRequests() { + for { + select { + case <-y.Quit(): + return + case req := <-y.RequestChan(): + if y.Config().IsDomainInScope(req.Domain) { + y.executeQuery(req.Domain) + } + } } } diff --git a/amass/wildcards.go b/amass/wildcards.go index 6a3e3ff88..d4a5afe81 100644 --- a/amass/wildcards.go +++ b/amass/wildcards.go @@ -4,12 +4,14 @@ package amass import ( + "errors" + "math/rand" "strings" "sync" "time" "github.com/OWASP/Amass/amass/core" - "github.com/google/uuid" + "github.com/miekg/dns" ) const ( @@ -17,7 +19,9 @@ const ( maxDNSNameLen = 253 maxDNSLabelLen = 63 + minLabelLen = 6 maxLabelLen = 24 + ldhChars = "abcdefghijklmnopqrstuvwxyz0123456789-" ) // Names for the different types of wildcards that can be detected. @@ -57,18 +61,18 @@ func GetWildcardType(req *core.Request) int { func performWildcardRequest(req *core.Request) int { base := len(strings.Split(req.Domain, ".")) - labels := strings.Split(req.Name, ".") + labels := strings.Split(strings.ToLower(req.Name), ".") + if len(labels) > base { + labels = labels[1:] + } - for i := len(labels) - base; i > 0; i-- { - sub := strings.Join(labels[i:], ".") - w := getWildcard(sub) + for i := len(labels) - base; i >= 0; i-- { + w := getWildcard(strings.Join(labels[i:], ".")) if w.WildcardType == WildcardTypeDynamic { return WildcardTypeDynamic } else if w.WildcardType == WildcardTypeStatic { - if len(req.Records) == 0 { - return WildcardTypeStatic - } else if compareAnswers(req.Records, w.Answers) { + if len(req.Records) == 0 || compareAnswers(req.Records, w.Answers) { return WildcardTypeStatic } } @@ -89,21 +93,26 @@ func getWildcard(sub string) *wildcard { wildcards[sub] = entry test = true entry.Lock() - defer entry.Unlock() } wildcardLock.Unlock() // Check if the subdomain name is still be tested for a wildcard if !test { entry.RLock() - defer entry.RUnlock() + entry.RUnlock() return entry } // Query multiple times with unlikely names against this subdomain set := make([][]core.DNSAnswer, numOfWildcardTests) for i := 0; i < numOfWildcardTests; i++ { - a := wildcardTest(sub) - if a == nil { + a, err := wildcardTest(sub) + if err != nil { + // A test error gives it the most severe wildcard type + entry.WildcardType = WildcardTypeDynamic + entry.Unlock() + return entry + } else if a == nil { // There is no DNS wildcard + entry.Unlock() return entry } set[i] = a @@ -123,30 +132,39 @@ func getWildcard(sub string) *wildcard { } else { entry.WildcardType = WildcardTypeDynamic } + entry.Unlock() return entry } -func wildcardTest(sub string) []core.DNSAnswer { - var answers []core.DNSAnswer +var wildcardQueryTypes = []string{ + "CNAME", + "A", + "AAAA", +} +func wildcardTest(sub string) ([]core.DNSAnswer, error) { name := UnlikelyName(sub) if name == "" { - return nil - } - // Check if the name resolves - if a, err := Resolve(name, "CNAME"); err == nil { - answers = append(answers, a...) + return nil, errors.New("Failed to generate the unlikely name for DNS wildcard testing") } - if a, err := Resolve(name, "A"); err == nil { - answers = append(answers, a...) - } - if a, err := Resolve(name, "AAAA"); err == nil { - answers = append(answers, a...) + + var answers []core.DNSAnswer + for _, t := range wildcardQueryTypes { + if a, err := Resolve(name, t); err == nil { + if a != nil && len(a) > 0 { + answers = append(answers, a...) + } + } else if (err.(*resolveError)).Rcode == 100 || + (err.(*resolveError)).Rcode == dns.RcodeRefused || + (err.(*resolveError)).Rcode == dns.RcodeServerFailure || + (err.(*resolveError)).Rcode == dns.RcodeNotImplemented { + return nil, errors.New("Failed to get a DNS server response during wildcard testing") + } } if len(answers) == 0 { - return nil + return nil, nil } - return answers + return answers, nil } func compareAnswers(ans1, ans2 []core.DNSAnswer) bool { @@ -162,19 +180,31 @@ func compareAnswers(ans1, ans2 []core.DNSAnswer) bool { // UnlikelyName takes a subdomain name and returns an unlikely DNS name within that subdomain. func UnlikelyName(sub string) string { - newlabel := uuid.New().String() + var newlabel string + ldh := []rune(ldhChars) + ldhLen := len(ldh) // Determine the max label length l := maxDNSNameLen - (len(sub) + 1) if l > maxLabelLen { l = maxLabelLen - } else if l < 1 { + } else if l < minLabelLen { return "" } - if len(newlabel) > l { - newlabel = newlabel[:l] + // Shuffle our LDH characters + rand.Shuffle(ldhLen, func(i, j int) { + ldh[i], ldh[j] = ldh[j], ldh[i] + }) + + l = minLabelLen + rand.Intn((l-minLabelLen)+1) + for i := 0; i < l; i++ { + sel := rand.Int() % ldhLen + + newlabel = newlabel + string(ldh[sel]) + } + + if newlabel == "" { + return newlabel } - // Remove hyphens from the beginning and end of the label - newlabel = strings.Trim(newlabel, "-") - return newlabel + "." + sub + return strings.Trim(newlabel, "-") + "." + sub } diff --git a/cmd/amass.tracker/main.go b/cmd/amass.tracker/main.go index bbcbd38fc..1b342f62c 100644 --- a/cmd/amass.tracker/main.go +++ b/cmd/amass.tracker/main.go @@ -37,13 +37,14 @@ var ( green = color.New(color.FgHiGreen).SprintFunc() blue = color.New(color.FgHiBlue).SprintFunc() // Command-line switches and provided parameters - help = flag.Bool("h", false, "Show the program usage message") - list = flag.Bool("list", false, "Print information for all available enumerations") - vprint = flag.Bool("version", false, "Print the version number of this Amass binary") - dir = flag.String("dir", "", "Path to the directory containing the graph database") - last = flag.Int("last", 0, "The number of recent enumerations to include in the tracking") - startStr = flag.String("start", "", "Exclude all enumerations before (format: "+timeFormat+")") - history = flag.Bool("history", false, "Show the difference between all enumeration pairs") + help = flag.Bool("h", false, "Show the program usage message") + list = flag.Bool("list", false, "Print information for all available enumerations") + vprint = flag.Bool("version", false, "Print the version number of this Amass binary") + dir = flag.String("dir", "", "Path to the directory containing the graph database") + last = flag.Int("last", 0, "The number of recent enumerations to include in the tracking") + startStr = flag.String("start", "", "Exclude all enumerations before (format: "+timeFormat+")") + history = flag.Bool("history", false, "Show the difference between all enumeration pairs") + domainspath = flag.String("df", "", "Path to a file providing root domain names") ) func main() { @@ -69,10 +70,6 @@ func main() { fmt.Fprintf(color.Error, "version %s\n", amass.Version) os.Exit(1) } - if len(domains) == 0 { - r.Fprintln(color.Error, "No root domain names were provided") - os.Exit(1) - } if *startStr != "" && *last != 0 { r.Fprintln(color.Error, "The start flag cannot be used with the last or all flags") os.Exit(1) @@ -81,6 +78,18 @@ func main() { r.Fprintln(color.Error, "Tracking requires more than one enumeration") os.Exit(1) } + if *domainspath != "" { + list, err := core.GetListFromFile(*domainspath) + if err != nil { + r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err) + os.Exit(1) + } + domains = utils.UniqueAppend(domains, list...) + } + if len(domains) == 0 { + r.Fprintln(color.Error, "No root domain names were provided") + os.Exit(1) + } var err error var start time.Time @@ -113,8 +122,11 @@ func main() { var enums []string // Obtain the enumerations that include the provided domain for _, e := range graph.EnumerationList() { - if enumContainsDomain(e, domains[0], graph) { - enums = append(enums, e) + for _, domain := range domains { + if enumContainsDomain(e, domain, graph) { + enums = append(enums, e) + break + } } } @@ -159,19 +171,19 @@ func main() { } if *history { - completeHistoryOutput(domains[0], enums, earliest, latest, graph) + completeHistoryOutput(domains, enums, earliest, latest, graph) return } - cumulativeOutput(domains[0], enums, earliest, latest, graph) + cumulativeOutput(domains, enums, earliest, latest, graph) } -func cumulativeOutput(domain string, enums []string, ea, la []time.Time, h handlers.DataHandler) { +func cumulativeOutput(domains []string, enums []string, ea, la []time.Time, h handlers.DataHandler) { idx := len(enums) - 1 filter := utils.NewStringFilter() var cum []*core.Output for i := idx - 1; i >= 0; i-- { - for _, out := range getEnumDataInScope(domain, enums[i], h) { + for _, out := range getEnumDataInScope(domains, enums[i], h) { if !filter.Duplicate(out.Name) { cum = append(cum, out) } @@ -185,8 +197,8 @@ func cumulativeOutput(domain string, enums []string, ea, la []time.Time, h handl blueLine() var updates bool - out := getEnumDataInScope(domain, enums[idx], h) - for _, d := range diffEnumOutput(domain, cum, out) { + out := getEnumDataInScope(domains, enums[idx], h) + for _, d := range diffEnumOutput(cum, out) { updates = true fmt.Fprintln(color.Output, d) } @@ -195,7 +207,7 @@ func cumulativeOutput(domain string, enums []string, ea, la []time.Time, h handl } } -func completeHistoryOutput(domain string, enums []string, ea, la []time.Time, h handlers.DataHandler) { +func completeHistoryOutput(domains []string, enums []string, ea, la []time.Time, h handlers.DataHandler) { var prev string for i, enum := range enums { @@ -214,9 +226,9 @@ func completeHistoryOutput(domain string, enums []string, ea, la []time.Time, h blueLine() var updates bool - out1 := getEnumDataInScope(domain, prev, h) - out2 := getEnumDataInScope(domain, enum, h) - for _, d := range diffEnumOutput(domain, out1, out2) { + out1 := getEnumDataInScope(domains, prev, h) + out2 := getEnumDataInScope(domains, enum, h) + for _, d := range diffEnumOutput(out1, out2) { updates = true fmt.Fprintln(color.Output, d) } @@ -234,40 +246,43 @@ func blueLine() { fmt.Println() } -func getEnumDataInScope(domain, enum string, h handlers.DataHandler) []*core.Output { +func getEnumDataInScope(domains []string, enum string, h handlers.DataHandler) []*core.Output { var out []*core.Output for _, o := range h.GetOutput(enum, true) { - if strings.HasSuffix(o.Name, domain) { - out = append(out, o) + for _, domain := range domains { + if strings.HasSuffix(o.Name, domain) { + out = append(out, o) + break + } } } return out } -func diffEnumOutput(domain string, eout1, eout2 []*core.Output) []string { - emap1 := make(map[string]*core.Output) - emap2 := make(map[string]*core.Output) +func diffEnumOutput(out1, out2 []*core.Output) []string { + omap1 := make(map[string]*core.Output) + omap2 := make(map[string]*core.Output) - for _, o := range eout1 { - emap1[o.Name] = o + for _, o := range out1 { + omap1[o.Name] = o } - for _, o := range eout2 { - emap2[o.Name] = o + for _, o := range out2 { + omap2[o.Name] = o } handled := make(map[string]struct{}) var diff []string - for _, o := range eout1 { + for _, o := range out1 { handled[o.Name] = struct{}{} - if _, found := emap2[o.Name]; !found { + if _, found := omap2[o.Name]; !found { diff = append(diff, fmt.Sprintf("%s%s %s", blue("Removed: "), green(o.Name), yellow(lineOfAddresses(o.Addresses)))) continue } - o2 := emap2[o.Name] + o2 := omap2[o.Name] if !compareAddresses(o.Addresses, o2.Addresses) { diff = append(diff, fmt.Sprintf("%s%s\n\t%s\t%s\n\t%s\t%s", blue("Moved: "), green(o.Name), blue(" from "), yellow(lineOfAddresses(o.Addresses)), @@ -275,12 +290,12 @@ func diffEnumOutput(domain string, eout1, eout2 []*core.Output) []string { } } - for _, o := range eout2 { + for _, o := range out2 { if _, found := handled[o.Name]; found { continue } - if _, found := emap1[o.Name]; !found { + if _, found := omap1[o.Name]; !found { diff = append(diff, fmt.Sprintf("%s%s %s", blue("Found: "), green(o.Name), yellow(lineOfAddresses(o.Addresses)))) }