diff --git a/datasrcs/scripting/dns.go b/datasrcs/scripting/dns.go index 2285850fb..8ed834fe8 100644 --- a/datasrcs/scripting/dns.go +++ b/datasrcs/scripting/dns.go @@ -10,6 +10,7 @@ import ( "fmt" "net" "strings" + "sync" "time" amassnet "github.com/OWASP/Amass/v3/net" @@ -17,10 +18,21 @@ import ( "github.com/OWASP/Amass/v3/requests" "github.com/caffix/resolve" "github.com/miekg/dns" + bf "github.com/tylertreat/BoomFilters" lua "github.com/yuin/gopher-lua" "golang.org/x/net/publicsuffix" ) +const ( + defaultSweepSize = 250 + activeSweepSize = 500 +) + +var ( + sweepLock sync.Mutex + sweepFilter *bf.StableBloomFilter = bf.NewDefaultStableBloomFilter(1000000, 0.01) +) + // Wrapper so that scripts can make DNS queries. func (s *Script) resolve(L *lua.LState) int { ctx, err := extractContext(L.CheckUserData(1)) @@ -139,6 +151,102 @@ func convertType(qtype string) uint16 { return t } +func (s *Script) reverseSweep(L *lua.LState) int { + ctx, err := extractContext(L.CheckUserData(1)) + if err != nil { + L.Push(lua.LString("failed to obtain the context")) + return 1 + } + + addr := L.CheckString(2) + if addr == "" { + L.Push(lua.LString("failed to obtain the IP address")) + return 1 + } + + size := defaultSweepSize + if s.sys.Config().Active { + size = activeSweepSize + } + + var cidr *net.IPNet + if asn := s.sys.Cache().AddrSearch(addr); asn != nil { + if _, c, err := net.ParseCIDR(asn.Prefix); err == nil { + cidr = c + } + } + + if cidr == nil { + ip := net.ParseIP(addr) + mask := net.CIDRMask(18, 32) + if amassnet.IsIPv6(ip) { + mask = net.CIDRMask(64, 128) + } + + cidr = &net.IPNet{ + IP: ip.Mask(mask), + Mask: mask, + } + } + + var count int + ch := make(chan *resolve.ExtractedAnswer, 10) + for _, ip := range amassnet.CIDRSubset(cidr, addr, size) { + select { + case <-ctx.Done(): + L.Push(lua.LString("the context expired")) + return 1 + default: + } + + sweepLock.Lock() + if a := ip.String(); !sweepFilter.TestAndAdd([]byte(a)) { + count++ + go s.getPTR(ctx, a, ch) + } + sweepLock.Unlock() + } + + var records []*resolve.ExtractedAnswer + for i := 0; i < count; i++ { + if rr := <-ch; rr != nil { + records = append(records, rr) + } + } + + for _, rr := range records { + s.newPTR(ctx, rr) + } + L.Push(lua.LNil) + return 1 +} + +func (s *Script) getPTR(ctx context.Context, addr string, ch chan *resolve.ExtractedAnswer) { + if reserved, _ := amassnet.IsReservedAddress(addr); reserved { + ch <- nil + return + } + + msg := resolve.ReverseMsg(addr) + resp, err := s.dnsQuery(ctx, msg, s.sys.Resolvers(), 10) + if err != nil || resp == nil { + ch <- nil + return + } + + resp, err = s.dnsQuery(ctx, msg, s.sys.TrustedResolvers(), 10) + if err != nil || resp == nil { + ch <- nil + return + } + + if ans := resolve.ExtractAnswers(resp); len(ans) > 0 { + if records := resolve.AnswersByType(ans, dns.TypePTR); len(records) > 0 { + ch <- records[0] + } + } +} + func (s *Script) zoneWalk(L *lua.LState) int { ctx, err := extractContext(L.CheckUserData(1)) if err != nil { @@ -231,7 +339,6 @@ func (s *Script) wrapZoneTransfer(L *lua.LState) int { entry.RawSetString("rrdata", lua.LString(rr.Data)) tb.Append(entry) } - // Zone Transfers can reveal DNS wildcards if n := amassdns.RemoveAsteriskLabel(req.Name); len(n) < len(req.Name) { // Signal the wildcard discovery diff --git a/datasrcs/scripting/new.go b/datasrcs/scripting/new.go index 7b8a861e6..bcb072ea0 100644 --- a/datasrcs/scripting/new.go +++ b/datasrcs/scripting/new.go @@ -7,13 +7,18 @@ package scripting import ( "context" "net" + "strings" "time" amassnet "github.com/OWASP/Amass/v3/net" + amassdns "github.com/OWASP/Amass/v3/net/dns" "github.com/OWASP/Amass/v3/net/http" "github.com/OWASP/Amass/v3/requests" + "github.com/caffix/resolve" + "github.com/miekg/dns" bf "github.com/tylertreat/BoomFilters" lua "github.com/yuin/gopher-lua" + "golang.org/x/net/publicsuffix" ) func (s *Script) genNewName(ctx context.Context, name string) { @@ -25,13 +30,12 @@ func (s *Script) newNameWithSrc(ctx context.Context, name, tag, src string) { select { case <-ctx.Done(): case <-s.Done(): - default: - s.Output() <- &requests.DNSRequest{ - Name: name, - Domain: domain, - Tag: tag, - Source: src, - } + case s.Output() <- &requests.DNSRequest{ + Name: name, + Domain: domain, + Tag: tag, + Source: src, + }: } } } @@ -127,18 +131,50 @@ func (s *Script) internalSendDNSRecords(ctx context.Context, name string, record select { case <-ctx.Done(): case <-s.Done(): - default: - s.Output() <- &requests.DNSRequest{ - Name: name, - Domain: domain, - Records: records, - Tag: s.Description(), - Source: s.String(), - } + case s.Output() <- &requests.DNSRequest{ + Name: name, + Domain: domain, + Records: records, + Tag: s.Description(), + Source: s.String(), + }: } } } +func (s *Script) newPTR(ctx context.Context, record *resolve.ExtractedAnswer) { + answer := strings.ToLower(resolve.RemoveLastDot(record.Data)) + if amassdns.RemoveAsteriskLabel(answer) != answer { + return + } + // Check that the name discovered is in scope + if d := s.sys.Config().WhichDomain(answer); d == "" { + return + } + + ptr := strings.ToLower(resolve.RemoveLastDot(record.Name)) + domain, err := publicsuffix.EffectiveTLDPlusOne(ptr) + if err != nil { + return + } + + select { + case <-ctx.Done(): + case <-s.Done(): + case s.Output() <- &requests.DNSRequest{ + Name: ptr, + Domain: domain, + Records: []requests.DNSAnswer{{ + Name: ptr, + Type: int(dns.TypePTR), + Data: answer, + }}, + Tag: s.Description(), + Source: s.String(), + }: + } +} + // Wrapper so that scripts can send discovered IP addresses to Amass. func (s *Script) newAddr(L *lua.LState) int { ip := net.ParseIP(L.CheckString(2)) @@ -155,13 +191,12 @@ func (s *Script) newAddr(L *lua.LState) int { select { case <-ctx.Done(): case <-s.Done(): - default: - s.Output() <- &requests.AddrRequest{ - Address: ip.String(), - Domain: domain, - Tag: s.SourceType, - Source: s.String(), - } + case s.Output() <- &requests.AddrRequest{ + Address: ip.String(), + Domain: domain, + Tag: s.SourceType, + Source: s.String(), + }: } } } @@ -232,13 +267,12 @@ func (s *Script) associated(L *lua.LState) int { select { case <-ctx.Done(): case <-s.Done(): - default: - s.Output() <- &requests.WhoisRequest{ - Domain: domain, - NewDomains: []string{assoc}, - Tag: s.SourceType, - Source: s.String(), - } + case s.Output() <- &requests.WhoisRequest{ + Domain: domain, + NewDomains: []string{assoc}, + Tag: s.SourceType, + Source: s.String(), + }: } } } diff --git a/datasrcs/scripting/script.go b/datasrcs/scripting/script.go index 354fa983d..0444e5466 100644 --- a/datasrcs/scripting/script.go +++ b/datasrcs/scripting/script.go @@ -123,6 +123,7 @@ func (s *Script) newLuaState(cfg *config.Config) *lua.LState { L.SetGlobal("scrape", L.NewFunction(s.scrape)) L.SetGlobal("crawl", L.NewFunction(s.crawl)) L.SetGlobal("resolve", L.NewFunction(s.resolve)) + L.SetGlobal("reverse_sweep", L.NewFunction(s.reverseSweep)) L.SetGlobal("zone_walk", L.NewFunction(s.zoneWalk)) L.SetGlobal("zone_transfer", L.NewFunction(s.wrapZoneTransfer)) L.SetGlobal("output_dir", L.NewFunction(s.outputdir)) diff --git a/enum/dns.go b/enum/dns.go index 2e4e9996f..9e1330e74 100644 --- a/enum/dns.go +++ b/enum/dns.go @@ -12,14 +12,11 @@ import ( "sync" "time" - amassnet "github.com/OWASP/Amass/v3/net" - amassdns "github.com/OWASP/Amass/v3/net/dns" "github.com/OWASP/Amass/v3/requests" "github.com/caffix/pipeline" "github.com/caffix/queue" "github.com/caffix/resolve" "github.com/miekg/dns" - "golang.org/x/net/publicsuffix" ) const ( @@ -136,6 +133,11 @@ func (dt *dnsTask) rootTaskFunc() pipeline.TaskFunc { if v.Domain != "" && v.Name == v.Domain { r = v.Clone().(*requests.DNSRequest) } + // send the PTR records straight to the store stage + if r != nil && (strings.HasSuffix(r.Name, ".in-addr.arpa") || strings.HasSuffix(r.Name, ".ip6.arpa")) { + pipeline.SendData(ctx, "store", r, tp) + return nil, nil + } case *requests.SubdomainRequest: r = &requests.DNSRequest{ Name: v.Name, @@ -186,24 +188,6 @@ func (dt *dnsTask) Process(ctx context.Context, data pipeline.Data, tp pipeline. } else { dt.enum.Config.Log.Printf("Failed to enter %s into the request registry on the %s DNS task", msg.Question[0].Name, dt.trust) } - case *requests.AddrRequest: - if reserved, _ := amassnet.IsReservedAddress(v.Address); !reserved { - msg := resolve.ReverseMsg(v.Address) - k := key(msg.Id, msg.Question[0].Name) - - if dt.addReqWithIncrement(k, &req{ - Ctx: ctx, - Data: data.Clone(), - Qtype: dns.TypePTR, - Attempts: 1, - InScope: v.InScope, - }) { - dt.pool.Query(ctx, msg, dt.resps) - return nil, nil - } else { - dt.enum.Config.Log.Printf("Failed to enter %s into the request registry on the %s DNS task", msg.Question[0].Name, dt.trust) - } - } } return data, nil } @@ -340,12 +324,6 @@ func (dt *dnsTask) processResp(resp *dns.Msg) { } else { go dt.retry(resolve.QueryMsg(v.Name, qtype), resp.Id, entry) } - case *requests.AddrRequest: - if resp.Rcode == dns.RcodeSuccess { - dt.processRevRequest(ctx, resp, name, qtype, v, entry) - } else { - go dt.retry(resolve.ReverseMsg(v.Address), resp.Id, entry) - } default: dt.delReqWithDecrement(k) } @@ -419,57 +397,6 @@ func (dt *dnsTask) processFwdRequest(ctx context.Context, resp *dns.Msg, name st dt.delReqWithDecrement(k) } -func (dt *dnsTask) processRevRequest(ctx context.Context, resp *dns.Msg, name string, qtype uint16, req *requests.AddrRequest, entry *req) { - defer dt.delReqWithDecrement(key(resp.Id, resp.Question[0].Name)) - - ans := resolve.ExtractAnswers(resp) - if len(ans) == 0 { - return - } - - rr := resolve.AnswersByType(ans, dns.TypePTR) - if len(rr) == 0 { - return - } - - if !dt.trusted { - dt.nextStage(ctx, req) - entry.Sent = true - return - } - - answer := strings.ToLower(resolve.RemoveLastDot(rr[0].Data)) - if amassdns.RemoveAsteriskLabel(answer) != answer { - return - } - // Check that the name discovered is in scope - d := dt.enum.Config.WhichDomain(answer) - if d == "" { - return - } - if re := dt.enum.Config.DomainRegex(d); re == nil || re.FindString(answer) != answer { - return - } - - ptr := strings.ToLower(resolve.RemoveLastDot(rr[0].Name)) - domain, err := publicsuffix.EffectiveTLDPlusOne(ptr) - if err != nil { - return - } - - dt.enum.nameSrc.newName(&requests.DNSRequest{ - Name: ptr, - Domain: domain, - Records: []requests.DNSAnswer{{ - Name: ptr, - Type: int(dns.TypePTR), - Data: answer, - }}, - Tag: requests.DNS, - Source: "Reverse DNS", - }) -} - func (dt *dnsTask) subdomainQueries(ctx context.Context, req *requests.DNSRequest, tp pipeline.TaskParams) { ch := make(chan []requests.DNSAnswer, 4) diff --git a/enum/enum.go b/enum/enum.go index fe9151e03..6268d66d3 100644 --- a/enum/enum.go +++ b/enum/enum.go @@ -6,6 +6,7 @@ package enum import ( "context" + "sync" "github.com/OWASP/Amass/v3/config" "github.com/OWASP/Amass/v3/datasrcs" @@ -31,6 +32,8 @@ type Enumeration struct { valTask *dnsTask store *dataManager requests queue.Queue + plock sync.Mutex + pending bool } // NewEnumeration returns an initialized Enumeration that has not been started yet. @@ -156,6 +159,7 @@ loop: if !ok { continue loop } + for name := range nameToSrc { if len(requestsMap[name]) == 0 && !pending[name] { go e.fireRequest(nameToSrc[name], element, finished) @@ -167,6 +171,7 @@ loop: case name := <-finished: if len(requestsMap[name]) == 0 { pending[name] = false + e.setRequestsPending(pending) continue loop } @@ -177,6 +182,28 @@ loop: e.requests.Process(func(e interface{}) {}) } +func (e *Enumeration) requestsPending() bool { + e.plock.Lock() + defer e.plock.Unlock() + + return e.pending +} + +func (e *Enumeration) setRequestsPending(p map[string]bool) { + var pending bool + + for _, b := range p { + if b { + pending = true + break + } + } + + e.plock.Lock() + e.pending = pending + e.plock.Unlock() +} + func (e *Enumeration) fireRequest(srv service.Service, req interface{}, finished chan string) { select { case <-e.done: diff --git a/enum/input.go b/enum/input.go index 8c9d75ea4..7a4af919b 100644 --- a/enum/input.go +++ b/enum/input.go @@ -6,10 +6,8 @@ package enum import ( "context" - "net" "regexp" "strconv" - "strings" "sync" "time" @@ -22,30 +20,24 @@ import ( bf "github.com/tylertreat/BoomFilters" ) -const ( - waitForDuration = 10 * time.Second - defaultSweepSize = 250 - activeSweepSize = 500 -) +const waitForDuration = 10 * time.Second // enumSource handles the filtering and release of new Data in the enumeration. type enumSource struct { - pipeline *pipeline.Pipeline - enum *Enumeration - queue queue.Queue - dups queue.Queue - sweeps queue.Queue - filter *bf.StableBloomFilter - sweepLock sync.Mutex - sweepFilter *bf.StableBloomFilter - subre *regexp.Regexp - done chan struct{} - doneOnce sync.Once - release chan struct{} - inputsig chan uint32 - max int - countLock sync.Mutex - count uint32 + pipeline *pipeline.Pipeline + enum *Enumeration + queue queue.Queue + dups queue.Queue + sweeps queue.Queue + filter *bf.StableBloomFilter + subre *regexp.Regexp + done chan struct{} + doneOnce sync.Once + release chan struct{} + inputsig chan uint32 + max int + countLock sync.Mutex + count uint32 } // newEnumSource returns an initialized input source for the enumeration pipeline. @@ -53,18 +45,17 @@ func newEnumSource(p *pipeline.Pipeline, e *Enumeration) *enumSource { size := e.Sys.TrustedResolvers().Len() * e.Config.TrustedQPS r := &enumSource{ - pipeline: p, - enum: e, - queue: queue.NewQueue(), - dups: queue.NewQueue(), - sweeps: queue.NewQueue(), - filter: bf.NewDefaultStableBloomFilter(1000000, 0.01), - sweepFilter: bf.NewDefaultStableBloomFilter(1000000, 0.01), - subre: dns.AnySubdomainRegex(), - done: make(chan struct{}), - release: make(chan struct{}, size), - inputsig: make(chan uint32, size*2), - max: size, + pipeline: p, + enum: e, + queue: queue.NewQueue(), + dups: queue.NewQueue(), + sweeps: queue.NewQueue(), + filter: bf.NewDefaultStableBloomFilter(1000000, 0.01), + subre: dns.AnySubdomainRegex(), + done: make(chan struct{}), + release: make(chan struct{}, size), + inputsig: make(chan uint32, size*2), + max: size, } // Monitor the enumeration for completion or termination go func() { @@ -93,7 +84,6 @@ func (r *enumSource) Stop() { r.dups.Process(func(e interface{}) {}) r.sweeps.Process(func(e interface{}) {}) r.filter.Reset() - r.sweepFilter.Reset() } func (r *enumSource) markDone() { @@ -110,28 +100,25 @@ func (r *enumSource) newName(req *requests.DNSRequest) { } if req.Name == "" || !req.Valid() { + r.releaseOutput(1) return } // Clean up the newly discovered name and domain requests.SanitizeDNSRequest(req) // Check that the name is valid if r.subre.FindString(req.Name) != req.Name { + r.releaseOutput(1) return } if r.enum.Config.Blacklisted(req.Name) { + r.releaseOutput(1) return } - // Do not further evaluate service subdomains - for _, label := range strings.Split(req.Name, ".") { - l := strings.ToLower(label) - - if l == "_tcp" || l == "_udp" || l == "_tls" { - return - } - } - if r.accept(req.Name, req.Tag, req.Source, true) { - r.queue.Append(req) + if !r.accept(req.Name, req.Tag, req.Source, true) { + r.releaseOutput(1) + return } + r.queue.Append(req) } func (r *enumSource) newAddr(req *requests.AddrRequest) { @@ -202,13 +189,15 @@ func (r *enumSource) Next(ctx context.Context) bool { r.markDone() return false case <-t.C: - if r.pipeline.DataItemCount() <= 0 { + if r.pipeline.DataItemCount() <= 0 && + !r.enum.requestsPending() && r.queue.Len() == 0 { r.markDone() return false } r.fillQueue() t.Reset(waitForDuration) case <-r.queue.Signal(): + t.Reset(waitForDuration) return true } } @@ -256,7 +245,6 @@ func (r *enumSource) fillQueue() { if fill := unfilled - len(r.release); fill > 0 { r.releaseOutput(fill) } - go r.requestSweeps() } } @@ -297,68 +285,6 @@ func (r *enumSource) monitorDataSrcOutput(srv service.Service) { } } -func (r *enumSource) requestSweeps() { - r.sweepLock.Lock() - defer r.sweepLock.Unlock() - - for { - if unfilled := r.max - r.queue.Len(); unfilled <= 0 { - break - } - if e, ok := r.sweeps.Next(); ok { - // Generate the additional addresses to sweep across - _ = r.sweepAddrs(r.enum.ctx, e.(*requests.AddrRequest)) - } - } -} - -func (r *enumSource) sweepAddrs(ctx context.Context, req *requests.AddrRequest) int { - size := defaultSweepSize - if r.enum.Config.Active { - size = activeSweepSize - } - - var count int - cidr := r.addrCIDR(req.Address) - for _, ip := range amassnet.CIDRSubset(cidr, req.Address, size) { - select { - case <-ctx.Done(): - return count - default: - } - - if a := ip.String(); !r.sweepFilter.TestAndAdd([]byte(a)) { - count++ - r.queue.Append(&requests.AddrRequest{ - Address: a, - Domain: req.Domain, - Tag: req.Tag, - Source: req.Source, - }) - } - } - return count -} - -func (r *enumSource) addrCIDR(addr string) *net.IPNet { - if asn := r.enum.Sys.Cache().AddrSearch(addr); asn != nil { - if _, cidr, err := net.ParseCIDR(asn.Prefix); err == nil { - return cidr - } - } - - ip := net.ParseIP(addr) - mask := net.CIDRMask(18, 32) - if amassnet.IsIPv6(ip) { - mask = net.CIDRMask(64, 128) - } - - return &net.IPNet{ - IP: ip.Mask(mask), - Mask: mask, - } -} - // This goroutine ensures that duplicate names from other sources are shown in the Graph. func (r *enumSource) processDupNames() { countdown := r.max * 2 diff --git a/enum/names.go b/enum/names.go index 1054502c4..841edbe27 100644 --- a/enum/names.go +++ b/enum/names.go @@ -87,7 +87,7 @@ func (r *subdomainTask) checkForSubdomains(ctx context.Context, req *requests.DN dlabels := strings.Split(req.Domain, ".") // It cannot have fewer labels than the root domain name if len(nlabels)-1 < len(dlabels) { - return false + return true } sub := strings.TrimSpace(strings.Join(nlabels[1:], ".")) @@ -99,10 +99,8 @@ func (r *subdomainTask) checkForSubdomains(ctx context.Context, req *requests.DN return false } else if times == 1 && r.enum.graph.IsCNAMENode(ctx, sub) { r.cnames.Insert(sub) - return false + return true } else if times > 1 && r.cnames.Has(sub) { - return false - } else if times > r.enum.Config.MinForRecursive { return true } diff --git a/enum/store.go b/enum/store.go index 06ff5119f..30f206295 100644 --- a/enum/store.go +++ b/enum/store.go @@ -112,28 +112,29 @@ func (dm *dataManager) dnsRequest(ctx context.Context, req *requests.DNSRequest, default: } + var e error switch uint16(r.Type) { case dns.TypeA: - err = dm.insertA(ctx, req, i, tp) + e = dm.insertA(ctx, req, i, tp) case dns.TypeAAAA: - err = dm.insertAAAA(ctx, req, i, tp) + e = dm.insertAAAA(ctx, req, i, tp) case dns.TypePTR: - err = dm.insertPTR(ctx, req, i, tp) + e = dm.insertPTR(ctx, req, i, tp) case dns.TypeSRV: - err = dm.insertSRV(ctx, req, i, tp) + e = dm.insertSRV(ctx, req, i, tp) case dns.TypeNS: - err = dm.insertNS(ctx, req, i, tp) + e = dm.insertNS(ctx, req, i, tp) case dns.TypeMX: - err = dm.insertMX(ctx, req, i, tp) + e = dm.insertMX(ctx, req, i, tp) case dns.TypeTXT: - err = dm.insertTXT(ctx, req, i, tp) + e = dm.insertTXT(ctx, req, i, tp) case dns.TypeSOA: - err = dm.insertSOA(ctx, req, i, tp) + e = dm.insertSOA(ctx, req, i, tp) case dns.TypeSPF: - err = dm.insertSPF(ctx, req, i, tp) + e = dm.insertSPF(ctx, req, i, tp) } - if err != nil { - break + if err == nil { + err = e } } return err @@ -214,8 +215,8 @@ func (dm *dataManager) insertPTR(ctx context.Context, req *requests.DNSRequest, dm.enum.nameSrc.newName(&requests.DNSRequest{ Name: target, Domain: domain, - Tag: requests.DNS, - Source: "Reverse DNS", + Tag: req.Tag, + Source: req.Source, }) if err := dm.enum.graph.UpsertPTR(ctx, req.Name, target, req.Source, dm.enum.Config.UUID.String()); err != nil { return fmt.Errorf("%s failed to insert PTR record: %v", dm.enum.graph, err) diff --git a/resources/scripts/dns/sweep.ads b/resources/scripts/dns/sweep.ads new file mode 100644 index 000000000..a0fb02f2a --- /dev/null +++ b/resources/scripts/dns/sweep.ads @@ -0,0 +1,28 @@ +-- Copyright © by Jeff Foley 2017-2023. All rights reserved. +-- Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. +-- SPDX-License-Identifier: Apache-2.0 + +name = "Reverse DNS" +type = "dns" + +local cfg + +function start() + cfg = config() +end + +function resolved(ctx, name, domain, records) + if (cfg == nil or cfg.mode == "passive") then + return + end + + if not in_scope(ctx, name) then + return + end + + for _, rec in pairs(records) do + if (rec.rrtype == 1 or rec.rrtype == 28) then + _ = reverse_sweep(ctx, rec.rrdata) + end + end +end