diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b7c11a1b6..e1462e92b 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -38,6 +38,24 @@ builds: - goos: windows goarch: 386 + - + main: ./cmd/amass.viz/main.go + binary: amass.viz + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - 386 + env: + - CGO_ENABLED=0 + ignore: + - goos: darwin + goarch: 386 + - goos: windows + goarch: 386 + - main: ./cmd/amass.db/main.go binary: amass.db @@ -55,10 +73,10 @@ builds: goarch: 386 - goos: windows goarch: 386 - + - - main: ./cmd/amass.viz/main.go - binary: amass.viz + main: ./cmd/amass.tracker/main.go + binary: amass.tracker goos: - windows - linux diff --git a/README.md b/README.md index 110f6e8f8..4611dd778 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,14 @@ The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping. Additionally, Amass uses the IP addresses obtained during resolution to discover associated netblocks and ASNs. All the information is then used to build maps of the target networks. +**Information gathering techniques used:** + +* DNS: Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request) +* Scraping: Ask, Baidu, Bing, CommonCrawl, DNSDB, DNSDumpster, DNSTable, Dogpile, Exalead, FindSubdomains, Google, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ThreatCrowd, VirusTotal, Yahoo +* Certificates: Active pulls (upon request), Censys, CertDB, CertSpotter, Crtsh, Entrust +* APIs: BinaryEdge, BufferOver, CIRCL, HackerTarget, PassiveTotal, Robtex, SecurityTrails, Shodan, Twitter, Umbrella, URLScan +* Web Archives: ArchiveIt, ArchiveToday, Arquivo, LoCArchive, OpenUKArchive, UKGovArchive, Wayback + ---- ## How to Install @@ -100,8 +108,8 @@ Join our Discord server: [![Chat on Discord](https://img.shields.io/discord/4337 [![Follow on Twitter](https://img.shields.io/twitter/follow/jeff_foley.svg?logo=twitter)](https://twitter.com/jeff_foley) -- OWASP: [Caffix](https://www.owasp.org/index.php/User:Caffix) -- GitHub: [@caffix](https://github.com/caffix) +* OWASP: [Caffix](https://www.owasp.org/index.php/User:Caffix) +* GitHub: [@caffix](https://github.com/caffix) ### Contributors @@ -116,27 +124,27 @@ This project improves thanks to all the people who contribute: ## Mentions -- [Pose a Threat: How Perceptual Analysis Helps Bug Hunters](https://www.bishopfox.com/news/2018/12/appsec-california-pose-a-threat-how-perpetual-analysis-helps-bug-hunters/) -- [A penetration tester’s guide to subdomain enumeration](https://blog.appsecco.com/a-penetration-testers-guide-to-sub-domain-enumeration-7d842d5570f6) -- [Abusing access control on a large online e-commerce site to register as supplier](https://medium.com/@fbotes2/governit-754becf85cbc) -- [Black Hat Training, Making the Cloud Rain Shells!: Discovery and Recon](https://www.blackhat.com/eu-18/training/schedule/index.html#aws--azure-exploitation-making-the-cloud-rain-shells-11060) -- [Subdomains Enumeration Cheat Sheet](https://pentester.land/cheatsheets/2018/11/14/subdomains-enumeration-cheatsheet.html) -- [Search subdomains and build graphs of network structure with Amass](https://miloserdov.org/?p=2309) -- [Getting started in Bug Bounty](https://medium.com/@ehsahil/getting-started-in-bug-bounty-7052da28445a) -- [Source code disclosure via exposed .git folder](https://pentester.land/tutorials/2018/10/25/source-code-disclosure-via-exposed-git-folder.html) -- [Amass, the best application to search for subdomains](https://www.h1rd.com/hacking/amass-para-buscar-subdominios) -- [Subdomain Takeover: Finding Candidates](https://0xpatrik.com/subdomain-takeover-candidates/) -- [Paul's Security Weekly #564: Technical Segment - Bug Bounty Hunting](https://wiki.securityweekly.com/Episode564) -- [The Bug Hunters Methodology v3(ish)](https://www.youtube.com/watch?v=Qw1nNPiH_Go) -- [Doing Recon the Correct Way](https://enciphers.com/doing-recon-the-correct-way/) -- [Discovering subdomains](https://www.sjoerdlangkemper.nl/2018/06/20/discovering-subdomains/) -- [Best Hacking Tools List for Hackers & Security Professionals 2018](http://kalilinuxtutorials.com/best-hacking-tools-list/amp/) -- [Amass - Subdomain Enumeration Tool](https://hydrasky.com/network-security/kali-tools/amass-subdomain-enumeration-tool/) -- [Subdomain enumeration](http://10degres.net/subdomain-enumeration/) -- [Asset Discovery: Doing Reconnaissance the Hard Way](https://0xpatrik.com/asset-discovery/) -- [Project Sonar: An Underrated Source of Internet-wide Data](https://0xpatrik.com/project-sonar-guide/) -- [Go is for everyone](https://changelog.com/gotime/71) -- [Top Five Ways the Red Team breached the External Perimeter](https://medium.com/@adam.toscher/top-five-ways-the-red-team-breached-the-external-perimeter-262f99dc9d17) +* [Pose a Threat: How Perceptual Analysis Helps Bug Hunters](https://www.bishopfox.com/news/2018/12/appsec-california-pose-a-threat-how-perpetual-analysis-helps-bug-hunters/) +* [A penetration tester’s guide to subdomain enumeration](https://blog.appsecco.com/a-penetration-testers-guide-to-sub-domain-enumeration-7d842d5570f6) +* [Abusing access control on a large online e-commerce site to register as supplier](https://medium.com/@fbotes2/governit-754becf85cbc) +* [Black Hat Training, Making the Cloud Rain Shells!: Discovery and Recon](https://www.blackhat.com/eu-18/training/schedule/index.html#aws--azure-exploitation-making-the-cloud-rain-shells-11060) +* [Subdomains Enumeration Cheat Sheet](https://pentester.land/cheatsheets/2018/11/14/subdomains-enumeration-cheatsheet.html) +* [Search subdomains and build graphs of network structure with Amass](https://miloserdov.org/?p=2309) +* [Getting started in Bug Bounty](https://medium.com/@ehsahil/getting-started-in-bug-bounty-7052da28445a) +* [Source code disclosure via exposed .git folder](https://pentester.land/tutorials/2018/10/25/source-code-disclosure-via-exposed-git-folder.html) +* [Amass, the best application to search for subdomains](https://www.h1rd.com/hacking/amass-para-buscar-subdominios) +* [Subdomain Takeover: Finding Candidates](https://0xpatrik.com/subdomain-takeover-candidates/) +* [Paul's Security Weekly #564: Technical Segment - Bug Bounty Hunting](https://wiki.securityweekly.com/Episode564) +* [The Bug Hunters Methodology v3(ish)](https://www.youtube.com/watch?v=Qw1nNPiH_Go) +* [Doing Recon the Correct Way](https://enciphers.com/doing-recon-the-correct-way/) +* [Discovering subdomains](https://www.sjoerdlangkemper.nl/2018/06/20/discovering-subdomains/) +* [Best Hacking Tools List for Hackers & Security Professionals 2018](http://kalilinuxtutorials.com/best-hacking-tools-list/amp/) +* [Amass - Subdomain Enumeration Tool](https://hydrasky.com/network-security/kali-tools/amass-subdomain-enumeration-tool/) +* [Subdomain enumeration](http://10degres.net/subdomain-enumeration/) +* [Asset Discovery: Doing Reconnaissance the Hard Way](https://0xpatrik.com/asset-discovery/) +* [Project Sonar: An Underrated Source of Internet-wide Data](https://0xpatrik.com/project-sonar-guide/) +* [Go is for everyone](https://changelog.com/gotime/71) +* [Top Five Ways the Red Team breached the External Perimeter](https://medium.com/@adam.toscher/top-five-ways-the-red-team-breached-the-external-perimeter-262f99dc9d17) ## Stargazers over Time diff --git a/amass/amass.go b/amass/amass.go index 06fdae138..6606978ea 100644 --- a/amass/amass.go +++ b/amass/amass.go @@ -151,6 +151,9 @@ func (e *Enumeration) Start() error { } } + // Use all previously discovered names that are in scope + go e.submitKnownNames() + var wg sync.WaitGroup wg.Add(2) go e.checkForOutput(&wg) @@ -190,6 +193,33 @@ loop: return nil } +func (e *Enumeration) submitKnownNames() { + for _, enum := range e.Graph.EnumerationList() { + var found bool + + for _, domain := range e.Graph.EnumerationDomains(enum) { + if e.Config.IsDomainInScope(domain) { + found = true + break + } + } + if !found { + continue + } + + for _, o := range e.Graph.GetOutput(enum, true) { + if e.Config.IsDomainInScope(o.Name) { + e.Bus.Publish(core.NewNameTopic, &core.Request{ + Name: o.Name, + Domain: o.Domain, + Tag: o.Tag, + Source: o.Source, + }) + } + } + } +} + // DNSQueriesPerSec returns the number of DNS queries the enumeration has performed per second. func (e *Enumeration) DNSQueriesPerSec() int { e.metricsLock.RLock() @@ -279,7 +309,7 @@ loop: case <-e.Done: break loop case <-t.C: - out := e.Graph.GetUnreadOutput(e.Config.UUID.String()) + out := e.Graph.GetOutput(e.Config.UUID.String(), false) for _, o := range out { if time.Now().Add(10 * time.Second).After(o.Timestamp) { e.Graph.MarkAsRead(&handlers.DataOptsParams{ @@ -296,7 +326,7 @@ loop: } } // Handle all remaining pieces of output - out := e.Graph.GetUnreadOutput(e.Config.UUID.String()) + out := e.Graph.GetOutput(e.Config.UUID.String(), false) for _, o := range out { if !e.filter.Duplicate(o.Name) { e.Graph.MarkAsRead(&handlers.DataOptsParams{ diff --git a/amass/brute.go b/amass/brute.go index d90b42bde..53c89ed76 100644 --- a/amass/brute.go +++ b/amass/brute.go @@ -36,7 +36,7 @@ type BruteForceService struct { // NewBruteForceService returns he object initialized, but not yet started. func NewBruteForceService(config *core.Config, bus *core.EventBus) *BruteForceService { bfs := &BruteForceService{ - max: utils.NewSimpleSemaphore(10000), + max: utils.NewSimpleSemaphore(5000), filter: utils.NewStringFilter(), } diff --git a/amass/dnssrv.go b/amass/dnssrv.go index 02290cc2f..3bcffccf2 100644 --- a/amass/dnssrv.go +++ b/amass/dnssrv.go @@ -48,7 +48,7 @@ type DNSService struct { // NewDNSService returns he object initialized, but not yet started. func NewDNSService(config *core.Config, bus *core.EventBus) *DNSService { ds := &DNSService{ - max: utils.NewSimpleSemaphore(10000), + max: utils.NewSimpleSemaphore(5000), filter: utils.NewStringFilter(), } diff --git a/amass/handlers/dataopts.go b/amass/handlers/dataopts.go index 82f9d9279..fc470ec2a 100644 --- a/amass/handlers/dataopts.go +++ b/amass/handlers/dataopts.go @@ -6,6 +6,7 @@ package handlers import ( "encoding/json" "io" + "time" "github.com/OWASP/Amass/amass/core" ) @@ -35,6 +36,26 @@ func (d *DataOptsHandler) Insert(data *DataOptsParams) error { return d.Enc.Encode(data) } +// EnumerationList returns a list of enumeration IDs found in the data. +func (d *DataOptsHandler) EnumerationList() []string { + return []string{} +} + +// EnumerationDomains returns the domains that were involved in the provided enumeration. +func (d *DataOptsHandler) EnumerationDomains(uuid string) []string { + return []string{} +} + +// EnumerationDateRange returns the date range associated with the provided enumeration UUID. +func (d *DataOptsHandler) EnumerationDateRange(uuid string) (time.Time, time.Time) { + return time.Now(), time.Now() +} + +// GetOutput implements the Amass DataHandler interface. +func (d *DataOptsHandler) GetOutput(uuid string, marked bool) []*core.Output { + return nil +} + // MarkAsRead implements the Amass DataHandler interface. func (d *DataOptsHandler) MarkAsRead(data *DataOptsParams) error { return nil @@ -44,8 +65,3 @@ func (d *DataOptsHandler) MarkAsRead(data *DataOptsParams) error { func (d *DataOptsHandler) IsCNAMENode(data *DataOptsParams) bool { return false } - -// GetUnreadOutput implements the Amass DataHandler interface. -func (d *DataOptsHandler) GetUnreadOutput(uuid string) []*core.Output { - return nil -} diff --git a/amass/handlers/graph.go b/amass/handlers/graph.go index 029469d67..3c6beccc2 100644 --- a/amass/handlers/graph.go +++ b/amass/handlers/graph.go @@ -23,6 +23,11 @@ import ( "github.com/cayleygraph/cayley/quad" ) +const ( + // DefaultGraphDBDirectory is the directory name used by default for the graph database. + DefaultGraphDBDirectory string = "amass_output" +) + // Graph is the object for managing a network infrastructure link graph. type Graph struct { sync.Mutex @@ -34,13 +39,13 @@ type Graph struct { func NewGraph(path string) *Graph { var err error - // If not directory was specified, $PWD/.amass/ will be used + // If a directory was not specified, $PWD/amass_output/ will be used if path == "" { path, err = os.Getwd() if err != nil { return nil } - path = filepath.Join(path, "amass") + path = filepath.Join(path, DefaultGraphDBDirectory) } // If the directory does not yet exist, create it if err = os.MkdirAll(path, 0755); err != nil { @@ -81,6 +86,34 @@ func (g *Graph) Close() { g.store.Close() } +// String implements the Amass data handler interface. +func (g *Graph) String() string { + return "Amass Graph" +} + +func (g *Graph) propertyValue(node quad.Value, pname, uuid string) string { + if quad.ToString(node) == "" || pname == "" || uuid == "" { + return "" + } + + p := cayley.StartPath(g.store, node).LabelContext(quad.String(uuid)).Out(quad.String(pname)) + it, _ := p.BuildIterator().Optimize() + it, _ = g.store.OptimizeIterator(it) + defer it.Close() + + var result string + ctx := context.TODO() + for it.Next(ctx) { + token := it.Result() + value := g.store.NameOf(token) + result = quad.NativeOf(value).(string) + if result != "" { + break + } + } + return result +} + func (g *Graph) dumpGraph() string { var result string @@ -108,11 +141,6 @@ func (g *Graph) dumpGraph() string { return result } -// String implements the Amass data handler interface. -func (g *Graph) String() string { - return "Amass Graph" -} - // Insert implements the Amass DataHandler interface. func (g *Graph) Insert(data *DataOptsParams) error { g.Lock() @@ -144,107 +172,6 @@ func (g *Graph) Insert(data *DataOptsParams) error { return err } -// MarkAsRead implements the Amass DataHandler interface. -func (g *Graph) MarkAsRead(data *DataOptsParams) error { - g.Lock() - defer g.Unlock() - - if t := g.propertyValue(quad.String(data.Name), "type", data.UUID); t != "" { - g.store.AddQuad(quad.Make(data.Name, "read", "yes", data.UUID)) - } - return nil -} - -// IsCNAMENode implements the Amass DataHandler interface. -func (g *Graph) IsCNAMENode(data *DataOptsParams) bool { - g.Lock() - defer g.Unlock() - - if r := g.propertyValue(quad.String(data.Name), "cname_to", data.UUID); r != "" { - return true - } - return false -} - -// VizData returns the current state of the Graph as viz package Nodes and Edges. -func (g *Graph) VizData(uuid string) ([]viz.Node, []viz.Edge) { - g.Lock() - defer g.Unlock() - - var idx int - var nodes []viz.Node - rnodes := make(map[string]int) - p := cayley.StartPath(g.store).Has(quad.String("type")).Unique() - p.Iterate(nil).EachValue(nil, func(node quad.Value) { - label := quad.ToString(node) - if label == "" { - return - } - - var source string - t := g.propertyValue(node, "type", uuid) - title := t + ": " + label - - switch t { - case "subdomain": - source = g.propertyValue(node, "source", uuid) - case "domain": - source = g.propertyValue(node, "source", uuid) - case "ns": - source = g.propertyValue(node, "source", uuid) - case "mx": - source = g.propertyValue(node, "source", uuid) - case "as": - title = title + ", Desc: " + g.propertyValue(node, "description", uuid) - } - - rnodes[label] = idx - nodes = append(nodes, viz.Node{ - ID: idx, - Type: t, - Label: label, - Title: title, - Source: source, - }) - idx++ - }) - - var edges []viz.Edge - for _, n := range nodes { - // Obtain all the predicates for this node - var predicates []quad.Value - p = cayley.StartPath(g.store, quad.String(n.Label)).OutPredicates().Unique() - p.Iterate(nil).EachValue(nil, func(val quad.Value) { - predicates = append(predicates, val) - }) - // Create viz edges for graph edges leaving the node - for _, predicate := range predicates { - path := cayley.StartPath(g.store, quad.String(n.Label)).Out(predicate) - path.Iterate(nil).EachValue(nil, func(val quad.Value) { - var to string - pstr := quad.ToString(predicate) - - if pstr == "root_of" || pstr == "cname_to" || pstr == "a_to" || - pstr == "aaaa_to" || pstr == "ptr_to" || pstr == "service_for" || - pstr == "srv_to" || pstr == "ns_to" || pstr == "mx_to" || - pstr == "contains" || pstr == "has_prefix" { - to = quad.ToString(val) - } - if to == "" { - return - } - - edges = append(edges, viz.Edge{ - From: n.ID, - To: rnodes[to], - Title: pstr, - }) - }) - } - } - return nodes, edges -} - func (g *Graph) insertDomain(data *DataOptsParams) error { if data.Domain == "" { return errors.New("Graph: insertDomain: no domain name provided") @@ -517,8 +444,94 @@ func (g *Graph) insertInfrastructure(data *DataOptsParams) error { return nil } -// GetUnreadOutput returns new findings within the enumeration Graph. -func (g *Graph) GetUnreadOutput(uuid string) []*core.Output { +// EnumerationList returns a list of enumeration IDs found in the data. +func (g *Graph) EnumerationList() []string { + g.Lock() + defer g.Unlock() + + p := cayley.StartPath(g.store).Has(quad.String("type"), quad.String("domain")).Labels() + it, _ := p.BuildIterator().Optimize() + it, _ = g.store.OptimizeIterator(it) + defer it.Close() + + var ids []string + ctx := context.TODO() + for it.Next(ctx) { + token := it.Result() + value := g.store.NameOf(token) + label := quad.NativeOf(value).(string) + + if label != "" { + ids = utils.UniqueAppend(ids, label) + } + } + return ids +} + +// EnumerationDomains returns the domains that were involved in the provided enumeration. +func (g *Graph) EnumerationDomains(uuid string) []string { + g.Lock() + defer g.Unlock() + + p := cayley.StartPath(g.store).LabelContext( + quad.String(uuid)).Has(quad.String("type"), quad.String("domain")) + it, _ := p.BuildIterator().Optimize() + it, _ = g.store.OptimizeIterator(it) + defer it.Close() + + var domains []string + ctx := context.TODO() + for it.Next(ctx) { + token := it.Result() + value := g.store.NameOf(token) + domain := quad.NativeOf(value).(string) + + if domain != "" { + domains = utils.UniqueAppend(domains, domain) + } + } + return domains +} + +// EnumerationDateRange returns the date range associated with the provided enumeration UUID. +func (g *Graph) EnumerationDateRange(uuid string) (time.Time, time.Time) { + g.Lock() + defer g.Unlock() + + p := cayley.StartPath(g.store).LabelContext(quad.String(uuid)).Out(quad.String("timestamp")) + it, _ := p.BuildIterator().Optimize() + it, _ = g.store.OptimizeIterator(it) + defer it.Close() + + first := true + var earliest, latest time.Time + ctx := context.TODO() + for it.Next(ctx) { + token := it.Result() + value := g.store.NameOf(token) + timestamp := quad.NativeOf(value).(string) + tt, err := time.Parse(time.RFC3339, timestamp) + if err != nil { + continue + } + if first { + earliest = tt + latest = tt + first = false + continue + } + if tt.Before(earliest) { + earliest = tt + } + if tt.After(latest) { + latest = tt + } + } + return earliest, latest +} + +// GetOutput returns new findings within the enumeration Graph. +func (g *Graph) GetOutput(uuid string, marked bool) []*core.Output { g.Lock() defer g.Unlock() @@ -535,7 +548,7 @@ func (g *Graph) GetUnreadOutput(uuid string) []*core.Output { value := g.store.NameOf(token) domain := quad.NativeOf(value).(string) - names := g.getSubdomainNames(domain, uuid) + names := g.getSubdomainNames(domain, uuid, marked) for _, name := range names { if o := g.buildOutput(name, uuid); o != nil { o.Domain = domain @@ -546,7 +559,7 @@ func (g *Graph) GetUnreadOutput(uuid string) []*core.Output { return results } -func (g *Graph) getSubdomainNames(domain, uuid string) []string { +func (g *Graph) getSubdomainNames(domain, uuid string, marked bool) []string { names := []string{domain} d := quad.String(domain) @@ -556,11 +569,15 @@ func (g *Graph) getSubdomainNames(domain, uuid string) []string { s := quad.String("subdomain") ns := quad.String("ns") mx := quad.String("mx") - // This path identifies the names that have been marked as 'read' - read := cayley.StartPath(g.store, d).LabelContext(u).Out(root).Has( - t, s, ns, mx).Has(quad.String("read"), quad.String("yes")) - // All the DNS name related nodes that have not already been read - p := cayley.StartPath(g.store, d).LabelContext(u).Out(root).Has(t, s, ns, mx).Except(read) + + p := cayley.StartPath(g.store, d).LabelContext(u).Out(root).Has(t, s, ns, mx) + if !marked { + // This path identifies the names that have been marked as 'read' + read := cayley.StartPath(g.store, d).LabelContext(u).Out(root).Has( + t, s, ns, mx).Has(quad.String("read"), quad.String("yes")) + // All the DNS name related nodes that have not already been read + p = p.Except(read) + } it, _ := p.BuildIterator().Optimize() it, _ = g.store.OptimizeIterator(it) defer it.Close() @@ -713,25 +730,103 @@ func (g *Graph) buildAddrInfo(addr, uuid string) *core.AddressInfo { return ainfo } -func (g *Graph) propertyValue(node quad.Value, pname, uuid string) string { - if quad.ToString(node) == "" || pname == "" || uuid == "" { - return "" +// MarkAsRead implements the Amass DataHandler interface. +func (g *Graph) MarkAsRead(data *DataOptsParams) error { + g.Lock() + defer g.Unlock() + + if t := g.propertyValue(quad.String(data.Name), "type", data.UUID); t != "" { + g.store.AddQuad(quad.Make(data.Name, "read", "yes", data.UUID)) } + return nil +} - p := cayley.StartPath(g.store, node).LabelContext(quad.String(uuid)).Out(quad.String(pname)) - it, _ := p.BuildIterator().Optimize() - it, _ = g.store.OptimizeIterator(it) - defer it.Close() +// IsCNAMENode implements the Amass DataHandler interface. +func (g *Graph) IsCNAMENode(data *DataOptsParams) bool { + g.Lock() + defer g.Unlock() - var result string - ctx := context.TODO() - for it.Next(ctx) { - token := it.Result() - value := g.store.NameOf(token) - result = quad.NativeOf(value).(string) - if result != "" { - break + if r := g.propertyValue(quad.String(data.Name), "cname_to", data.UUID); r != "" { + return true + } + return false +} + +// VizData returns the current state of the Graph as viz package Nodes and Edges. +func (g *Graph) VizData(uuid string) ([]viz.Node, []viz.Edge) { + g.Lock() + defer g.Unlock() + + var idx int + var nodes []viz.Node + rnodes := make(map[string]int) + p := cayley.StartPath(g.store).Has(quad.String("type")).Unique() + p.Iterate(nil).EachValue(nil, func(node quad.Value) { + label := quad.ToString(node) + if label == "" { + return + } + + var source string + t := g.propertyValue(node, "type", uuid) + title := t + ": " + label + + switch t { + case "subdomain": + source = g.propertyValue(node, "source", uuid) + case "domain": + source = g.propertyValue(node, "source", uuid) + case "ns": + source = g.propertyValue(node, "source", uuid) + case "mx": + source = g.propertyValue(node, "source", uuid) + case "as": + title = title + ", Desc: " + g.propertyValue(node, "description", uuid) + } + + rnodes[label] = idx + nodes = append(nodes, viz.Node{ + ID: idx, + Type: t, + Label: label, + Title: title, + Source: source, + }) + idx++ + }) + + var edges []viz.Edge + for _, n := range nodes { + // Obtain all the predicates for this node + var predicates []quad.Value + p = cayley.StartPath(g.store, quad.String(n.Label)).OutPredicates().Unique() + p.Iterate(nil).EachValue(nil, func(val quad.Value) { + predicates = append(predicates, val) + }) + // Create viz edges for graph edges leaving the node + for _, predicate := range predicates { + path := cayley.StartPath(g.store, quad.String(n.Label)).Out(predicate) + path.Iterate(nil).EachValue(nil, func(val quad.Value) { + var to string + pstr := quad.ToString(predicate) + + if pstr == "root_of" || pstr == "cname_to" || pstr == "a_to" || + pstr == "aaaa_to" || pstr == "ptr_to" || pstr == "service_for" || + pstr == "srv_to" || pstr == "ns_to" || pstr == "mx_to" || + pstr == "contains" || pstr == "has_prefix" { + to = quad.ToString(val) + } + if to == "" { + return + } + + edges = append(edges, viz.Edge{ + From: n.ID, + To: rnodes[to], + Title: pstr, + }) + }) } } - return result + return nodes, edges } diff --git a/amass/handlers/gremlin.go b/amass/handlers/gremlin.go index 032e690fe..e81da0064 100644 --- a/amass/handlers/gremlin.go +++ b/amass/handlers/gremlin.go @@ -84,77 +84,6 @@ func (g *Gremlin) String() string { return "Gremlin TinkerPop Handler" } -// MarkAsRead implements the Amass DataHandler interface. -func (g *Gremlin) MarkAsRead(data *DataOptsParams) error { - g.avail.Acquire(1) - defer g.avail.Release(1) - - bindings := map[string]string{ - "uuid": data.UUID, - "name": data.Name, - "domain": data.Domain, - } - - conn, err := g.pool.Get() - if err != nil { - return err - } - defer conn.Close() - - _, err = conn.Client.Execute( - // Find the domain name for the vertex - "g.V().hasLabel('domain').has('name', domain).has('enum', uuid).out('root_of')."+ - // Find the subdomain name related vertex in the graph - "hasLabel('domain','subdomain','ns','mx').has('name', name).has('enum', uuid)."+ - // Mark the subdomain name related vertex as read - "property('read', 'yes')", - bindings, - map[string]string{}, - ) - return err -} - -// IsCNAMENode implements the Amass DataHandler interface. -func (g *Gremlin) IsCNAMENode(data *DataOptsParams) bool { - g.avail.Acquire(1) - defer g.avail.Release(1) - - bindings := map[string]string{ - "uuid": data.UUID, - "name": data.Name, - "domain": data.Domain, - } - - conn, err := g.pool.Get() - if err != nil { - return false - } - defer conn.Close() - - resp, err := conn.Client.Execute( - // Find the vertex in the graph and determine if it is a CNAME - "g.V().hasLabel('subdomain').has('name', name).has('enum', uuid).outE('cname_to').count()", - bindings, - map[string]string{}, - ) - if err != nil { - return false - } - - b, err := json.Marshal(resp) - if err != nil { - return false - } - - var count [1][]int64 - if err = json.Unmarshal(b, &count); err == nil { - if len(count[0]) > 0 && count[0][0] > 0 { - return true - } - } - return false -} - type gremlinRequest struct { Params *DataOptsParams Err chan error @@ -721,8 +650,23 @@ func (g *Gremlin) insertInfrastructure(data *DataOptsParams) error { return err } -// GetUnreadOutput implements the Amass DataHandler interface. -func (g *Gremlin) GetUnreadOutput(uuid string) []*core.Output { +// EnumerationList returns a list of enumeration IDs found in the data. +func (g *Gremlin) EnumerationList() []string { + return []string{} +} + +// EnumerationDomains returns the domains that were involved in the provided enumeration. +func (g *Gremlin) EnumerationDomains(uuid string) []string { + return []string{} +} + +// EnumerationDateRange returns the date range associated with the provided enumeration UUID. +func (g *Gremlin) EnumerationDateRange(uuid string) (time.Time, time.Time) { + return time.Now(), time.Now() +} + +// GetOutput implements the Amass DataHandler interface. +func (g *Gremlin) GetOutput(uuid string, marked bool) []*core.Output { g.avail.Acquire(1) defer g.avail.Release(1) @@ -734,23 +678,25 @@ func (g *Gremlin) GetUnreadOutput(uuid string) []*core.Output { } defer conn.Close() - resp, err := conn.Client.Execute( - // Find the vertices connected to all the domain names - "g.V().hasLabel('domain').has('enum', uuid).out('root_of')."+ - // We are only interested in the vertices not yet marked - "has('enum', uuid).not(has('read','yes'))."+ - // Traverse all the 'cname_to' and 'srv_to' edges - "until(outE('cname_to','srv_to').count().is(0).or().loops().is(10))."+ - "repeat(out('cname_to','srv_to'))."+ - // Traverse to the address vertices - "out('a_to','aaaa_to').hasLabel('address').has('enum', uuid)."+ - // Traverse to the netblock vertex - "in('contains').hasLabel('netblock').has('enum', uuid)."+ - // Complete the path by reaching the AS - "in('has_prefix').hasLabel('as').has('enum', uuid).path().by(valueMap())", - bindings, - map[string]string{}, - ) + // Find the vertices connected to all the domain names + query := "g.V().hasLabel('domain').has('enum', uuid).out('root_of').has('enum', uuid)." + + if !marked { + // We are only interested in the vertices not yet marked + query = query + "not(has('read','yes'))." + } + + // Traverse all the 'cname_to' and 'srv_to' edges + query = query + "until(outE('cname_to','srv_to').count().is(0).or().loops().is(10))." + + "repeat(out('cname_to','srv_to'))." + + // Traverse to the address vertices + "out('a_to','aaaa_to').hasLabel('address').has('enum', uuid)." + + // Traverse to the netblock vertex + "in('contains').hasLabel('netblock').has('enum', uuid)." + + // Complete the path by reaching the AS + "in('has_prefix').hasLabel('as').has('enum', uuid).path().by(valueMap())" + + resp, err := conn.Client.Execute(query, bindings, map[string]string{}) if err == nil { var output []*core.Output @@ -897,3 +843,74 @@ func propertiesToData(props map[string][]string) *DataOptsParams { } return data } + +// MarkAsRead implements the Amass DataHandler interface. +func (g *Gremlin) MarkAsRead(data *DataOptsParams) error { + g.avail.Acquire(1) + defer g.avail.Release(1) + + bindings := map[string]string{ + "uuid": data.UUID, + "name": data.Name, + "domain": data.Domain, + } + + conn, err := g.pool.Get() + if err != nil { + return err + } + defer conn.Close() + + _, err = conn.Client.Execute( + // Find the domain name for the vertex + "g.V().hasLabel('domain').has('name', domain).has('enum', uuid).out('root_of')."+ + // Find the subdomain name related vertex in the graph + "hasLabel('domain','subdomain','ns','mx').has('name', name).has('enum', uuid)."+ + // Mark the subdomain name related vertex as read + "property('read', 'yes')", + bindings, + map[string]string{}, + ) + return err +} + +// IsCNAMENode implements the Amass DataHandler interface. +func (g *Gremlin) IsCNAMENode(data *DataOptsParams) bool { + g.avail.Acquire(1) + defer g.avail.Release(1) + + bindings := map[string]string{ + "uuid": data.UUID, + "name": data.Name, + "domain": data.Domain, + } + + conn, err := g.pool.Get() + if err != nil { + return false + } + defer conn.Close() + + resp, err := conn.Client.Execute( + // Find the vertex in the graph and determine if it is a CNAME + "g.V().hasLabel('subdomain').has('name', name).has('enum', uuid).outE('cname_to').count()", + bindings, + map[string]string{}, + ) + if err != nil { + return false + } + + b, err := json.Marshal(resp) + if err != nil { + return false + } + + var count [1][]int64 + if err = json.Unmarshal(b, &count); err == nil { + if len(count[0]) > 0 && count[0][0] > 0 { + return true + } + } + return false +} diff --git a/amass/handlers/handlers.go b/amass/handlers/handlers.go index 36b815c40..55316fbbf 100644 --- a/amass/handlers/handlers.go +++ b/amass/handlers/handlers.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "time" "github.com/OWASP/Amass/amass/core" ) @@ -62,16 +63,25 @@ type DataHandler interface { // Inserts data operations into the graph. Insert(data *DataOptsParams) error + // Returns a list of enumeration IDs found in the data. + EnumerationList() []string + + // Returns the domains that were involved in the provided enumeration. + EnumerationDomains(uuid string) []string + + // Returns the date range associated with the provided enumeration UUID. + EnumerationDateRange(uuid string) (time.Time, time.Time) + + // Returns complete paths in the graph, with the option of only unmarked results. + GetOutput(uuid string, marked bool) []*core.Output + // Sets a 'read' property on the vertex matching Name, Domain and UUID. MarkAsRead(data *DataOptsParams) error // Return true if the Name, Domain and UUID match a CNAME in the graph. IsCNAMENode(data *DataOptsParams) bool - // Returns complete paths in the graph that have not yet been marked. - GetUnreadOutput(uuid string) []*core.Output - - // Signals the handler to prepare for closing + // Signals the handler to prepare for closing. Close() } diff --git a/amass/handlers/neo4j.go b/amass/handlers/neo4j.go index bbb253b2a..1cee05e7f 100644 --- a/amass/handlers/neo4j.go +++ b/amass/handlers/neo4j.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "sync" + "time" "github.com/OWASP/Amass/amass/core" bolt "github.com/johnnadratowski/golang-neo4j-bolt-driver" @@ -49,39 +50,6 @@ func (n *Neo4j) String() string { return "Neo4j Database Handler" } -// MarkAsRead implements the Amass DataHandler interface. -func (n *Neo4j) MarkAsRead(data *DataOptsParams) error { - params := map[string]interface{}{ - "uuid": data.UUID, - "name": data.Name, - "domain": data.Domain, - } - - for _, label := range []string{"domain", "subdomain", "ns", "mx"} { - n.conn.ExecNeo("MATCH (domain:domain {name: {domain}, enum: {uuid}}) "+ - "MATCH (target:"+label+" {name: {name}, enum: {uuid}}) "+ - "MATCH (domain)-[:root_of]->(target) "+ - "SET target.read = 'yes'", params) - } - return nil -} - -// IsCNAMENode implements the Amass DataHandler interface. -func (n *Neo4j) IsCNAMENode(data *DataOptsParams) bool { - params := map[string]interface{}{ - "uuid": data.UUID, - "name": data.Name, - "domain": data.Domain, - } - - result, _ := n.conn.ExecNeo("MATCH (d:domain {name: {domain}, enum: {uuid}}) "+ - "MATCH (c:subdomain {name: {name}, enum: {uuid}}) "+ - "MATCH (d)-[:root_of]->(c)-[:cname_to]->(t) "+ - "RETURN count(t)", params) - fmt.Printf("%v\n", result.Metadata()) - return false -} - // Insert implements the Amass DataHandler interface. func (n *Neo4j) Insert(data *DataOptsParams) error { n.Lock() @@ -494,7 +462,55 @@ func (n *Neo4j) insertInfrastructure(data *DataOptsParams) error { return err } -// GetUnreadOutput implements the Amass DataHandler interface. -func (n *Neo4j) GetUnreadOutput(uuid string) []*core.Output { +// EnumerationList returns a list of enumeration IDs found in the data. +func (n *Neo4j) EnumerationList() []string { + return []string{} +} + +// EnumerationDomains returns the domains that were involved in the provided enumeration. +func (n *Neo4j) EnumerationDomains(uuid string) []string { + return []string{} +} + +// EnumerationDateRange returns the date range associated with the provided enumeration UUID. +func (n *Neo4j) EnumerationDateRange(uuid string) (time.Time, time.Time) { + return time.Now(), time.Now() +} + +// GetOutput implements the Amass DataHandler interface. +func (n *Neo4j) GetOutput(uuid string, marked bool) []*core.Output { + return nil +} + +// MarkAsRead implements the Amass DataHandler interface. +func (n *Neo4j) MarkAsRead(data *DataOptsParams) error { + params := map[string]interface{}{ + "uuid": data.UUID, + "name": data.Name, + "domain": data.Domain, + } + + for _, label := range []string{"domain", "subdomain", "ns", "mx"} { + n.conn.ExecNeo("MATCH (domain:domain {name: {domain}, enum: {uuid}}) "+ + "MATCH (target:"+label+" {name: {name}, enum: {uuid}}) "+ + "MATCH (domain)-[:root_of]->(target) "+ + "SET target.read = 'yes'", params) + } return nil } + +// IsCNAMENode implements the Amass DataHandler interface. +func (n *Neo4j) IsCNAMENode(data *DataOptsParams) bool { + params := map[string]interface{}{ + "uuid": data.UUID, + "name": data.Name, + "domain": data.Domain, + } + + result, _ := n.conn.ExecNeo("MATCH (d:domain {name: {domain}, enum: {uuid}}) "+ + "MATCH (c:subdomain {name: {name}, enum: {uuid}}) "+ + "MATCH (d)-[:root_of]->(c)-[:cname_to]->(t) "+ + "RETURN count(t)", params) + fmt.Printf("%v\n", result.Metadata()) + return false +} diff --git a/amass/resolvers.go b/amass/resolvers.go index b9308d71f..609e6ef6c 100644 --- a/amass/resolvers.go +++ b/amass/resolvers.go @@ -17,11 +17,6 @@ import ( "github.com/miekg/dns" ) -const ( - // ResolutionTimeout is the maximum time spent retrying a resolution request. - ResolutionTimeout time.Duration = 10 * time.Second -) - var ( // Public & free DNS servers publicResolvers = []string{ @@ -261,7 +256,9 @@ func (r *resolver) processMessage(msg *dns.Msg) { // Check that the query was successful if msg.Rcode != dns.RcodeSuccess { var again bool - if msg.Rcode == dns.RcodeRefused { + if msg.Rcode == dns.RcodeRefused || + msg.Rcode == dns.RcodeServerFailure || + msg.Rcode == dns.RcodeNotImplemented { again = true } r.returnRequest(req, &resolveResult{ @@ -453,19 +450,12 @@ func Resolve(name, qtype string) ([]core.DNSAnswer, error) { } var again bool - var attempts int - started := time.Now() var ans []core.DNSAnswer for { ans, again, err = nextResolver().resolve(name, qt) if !again { break } - - attempts++ - if attempts > 1 && time.Now().After(started.Add(ResolutionTimeout)) { - break - } } return ans, err } diff --git a/amass/utils/misc.go b/amass/utils/misc.go index 249ea3679..36a289788 100644 --- a/amass/utils/misc.go +++ b/amass/utils/misc.go @@ -31,7 +31,7 @@ type StringFilter struct { quit chan struct{} } -// NewStringFilter returns an initialized NameFilter. +// NewStringFilter returns an initialized StringFilter. func NewStringFilter() *StringFilter { sf := &StringFilter{ filter: cfilter.New(), diff --git a/amass/utils/network.go b/amass/utils/network.go index e3f475117..776c7617f 100644 --- a/amass/utils/network.go +++ b/amass/utils/network.go @@ -37,17 +37,17 @@ var ( func init() { jar, _ := cookiejar.New(nil) defaultClient = &http.Client{ - Timeout: 15 * time.Second, + Timeout: 30 * time.Second, Transport: &http.Transport{ DialContext: (&net.Dialer{ - Timeout: 15 * time.Second, - KeepAlive: 15 * time.Second, + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 200, IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 5 * time.Second, + TLSHandshakeTimeout: 20 * time.Second, + ExpectContinueTimeout: 20 * time.Second, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, Jar: jar, diff --git a/cmd/amass.tracker/main.go b/cmd/amass.tracker/main.go new file mode 100644 index 000000000..902d665ef --- /dev/null +++ b/cmd/amass.tracker/main.go @@ -0,0 +1,380 @@ +// Copyright 2017 Jeff Foley. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "fmt" + "math/rand" + "os" + "path" + "sort" + "strings" + "time" + + "github.com/OWASP/Amass/amass" + "github.com/OWASP/Amass/amass/core" + "github.com/OWASP/Amass/amass/handlers" + "github.com/OWASP/Amass/amass/utils" + "github.com/fatih/color" +) + +const ( + timeFormat string = "01/02 15:04:05 2006 MST" +) + +// Types that implement the flag.Value interface for parsing +type parseStrings []string + +var ( + // Colors used to ease the reading of program output + y = color.New(color.FgHiYellow) + g = color.New(color.FgHiGreen) + r = color.New(color.FgHiRed) + b = color.New(color.FgHiBlue) + fgR = color.New(color.FgRed) + fgY = color.New(color.FgYellow) + yellow = color.New(color.FgHiYellow).SprintFunc() + green = color.New(color.FgHiGreen).SprintFunc() + blue = color.New(color.FgHiBlue).SprintFunc() + // Command-line switches and provided parameters + help = flag.Bool("h", false, "Show the program usage message") + list = flag.Bool("list", false, "Print information for all available enumerations") + vprint = flag.Bool("version", false, "Print the version number of this Amass binary") + dir = flag.String("dir", "", "Path to the directory containing the graph database") + last = flag.Int("last", 0, "The number of recent enumerations to include in the tracking") + startStr = flag.String("start", "", "Exclude all enumerations before (format: "+timeFormat+")") + history = flag.Bool("history", false, "Show the difference between all enumeration pairs") +) + +func main() { + var domains parseStrings + + defaultBuf := new(bytes.Buffer) + flag.CommandLine.SetOutput(defaultBuf) + flag.Usage = func() { + printBanner() + g.Fprintf(color.Error, "Usage: %s [options] -d domain\n\n", path.Base(os.Args[0])) + flag.PrintDefaults() + g.Fprintln(color.Error, defaultBuf.String()) + } + flag.Var(&domains, "d", "Domain names separated by commas (can be used multiple times)") + flag.Parse() + + // Some input validation + if *help || len(os.Args) == 1 { + flag.Usage() + return + } + if *vprint { + fmt.Fprintf(color.Error, "version %s\n", amass.Version) + os.Exit(1) + } + if len(domains) == 0 { + r.Fprintln(color.Error, "No root domain names were provided") + os.Exit(1) + } + if *startStr != "" && *last != 0 { + r.Fprintln(color.Error, "The start flag cannot be used with the last or all flags") + os.Exit(1) + } + if *last == 1 { + r.Fprintln(color.Error, "Tracking requires more than one enumeration") + os.Exit(1) + } + + var err error + var start time.Time + if *startStr != "" { + start, err = time.Parse(timeFormat, *startStr) + if err != nil { + r.Fprintf(color.Error, "%s is not in the correct format: %s\n", *startStr, timeFormat) + os.Exit(1) + } + } + + rand.Seed(time.Now().UTC().UnixNano()) + // Check that the default graph database directory exists in the CWD + if *dir == "" { + if finfo, err := os.Stat(handlers.DefaultGraphDBDirectory); os.IsNotExist(err) || !finfo.IsDir() { + r.Fprintln(color.Error, "Failed to open the graph database") + os.Exit(1) + } + } else if finfo, err := os.Stat(*dir); os.IsNotExist(err) || !finfo.IsDir() { + r.Fprintln(color.Error, "Failed to open the graph database") + os.Exit(1) + } + + graph := handlers.NewGraph(*dir) + if graph == nil { + r.Fprintln(color.Error, "Failed to open the graph database") + os.Exit(1) + } + + var enums []string + // Obtain the enumerations that include the provided domain + for _, e := range graph.EnumerationList() { + if enumContainsDomain(e, domains[0], graph) { + enums = append(enums, e) + } + } + + // The default is to use all the enumerations available + if *last == 0 { + *last = len(enums) + } + + var begin int + enums, earliest, latest := orderedEnumsAndDateRanges(enums, graph) + // Filter out enumerations that begin before the start date/time + if *startStr != "" { + for _, e := range earliest { + if !e.Before(start) { + break + } + begin++ + } + } else { // Or the number of enumerations from the end of the timeline + if len(enums) < *last { + r.Fprintf(color.Error, "%d enumerations are not available\n", *last) + os.Exit(1) + } + + begin = len(enums) - *last + } + enums = enums[begin:] + earliest = earliest[begin:] + latest = latest[begin:] + + // Check if the user has requested the list of enumerations + if *list { + for i := range enums { + g.Printf("%d) %s -> %s\n", i+1, earliest[i].Format(timeFormat), latest[i].Format(timeFormat)) + } + return + } + + if *history { + completeHistoryOutput(domains[0], enums, earliest, latest, graph) + return + } + cumulativeOutput(domains[0], enums, earliest, latest, graph) +} + +func cumulativeOutput(domain string, enums []string, ea, la []time.Time, h handlers.DataHandler) { + idx := len(enums) - 1 + filter := utils.NewStringFilter() + + var cum []*core.Output + for i := idx - 1; i >= 0; i-- { + for _, out := range getEnumDataInScope(domain, enums[i], h) { + if !filter.Duplicate(out.Name) { + cum = append(cum, out) + } + } + } + + blueLine() + fmt.Fprintf(color.Output, "%s\t%s%s%s\n%s\t%s%s%s\n", blue("Between"), + yellow(ea[0].Format(timeFormat)), blue(" -> "), yellow(la[0].Format(timeFormat)), + blue("and"), yellow(ea[idx].Format(timeFormat)), blue(" -> "), yellow(la[idx].Format(timeFormat))) + blueLine() + + out := getEnumDataInScope(domain, enums[idx], h) + for _, d := range diffEnumOutput(domain, cum, out) { + fmt.Fprintln(color.Output, d) + } +} + +func completeHistoryOutput(domain string, enums []string, ea, la []time.Time, h handlers.DataHandler) { + var prev string + + for i, enum := range enums { + if prev == "" { + prev = enum + continue + } + if i != 1 { + fmt.Println() + } + + blueLine() + fmt.Fprintf(color.Output, "%s\t%s%s%s\n%s\t%s%s%s\n", blue("Between"), + yellow(ea[i-1].Format(timeFormat)), blue(" -> "), yellow(la[i-1].Format(timeFormat)), + blue("and"), yellow(ea[i].Format(timeFormat)), blue(" -> "), yellow(la[i].Format(timeFormat))) + blueLine() + + out1 := getEnumDataInScope(domain, prev, h) + out2 := getEnumDataInScope(domain, enum, h) + for _, d := range diffEnumOutput(domain, out1, out2) { + fmt.Fprintln(color.Output, d) + } + prev = enum + } +} + +func blueLine() { + for i := 0; i < 8; i++ { + b.Fprint(color.Output, "----------") + } + fmt.Println() +} + +func getEnumDataInScope(domain, enum string, h handlers.DataHandler) []*core.Output { + var out []*core.Output + + for _, o := range h.GetOutput(enum, true) { + if strings.HasSuffix(o.Name, domain) { + out = append(out, o) + } + } + return out +} + +func diffEnumOutput(domain string, eout1, eout2 []*core.Output) []string { + emap1 := make(map[string]*core.Output) + emap2 := make(map[string]*core.Output) + + for _, o := range eout1 { + emap1[o.Name] = o + } + for _, o := range eout2 { + emap2[o.Name] = o + } + + handled := make(map[string]struct{}) + var diff []string + for _, o := range eout1 { + handled[o.Name] = struct{}{} + + if _, found := emap2[o.Name]; !found { + diff = append(diff, fmt.Sprintf("%s%s %s", blue("Removed: "), + green(o.Name), yellow(lineOfAddresses(o.Addresses)))) + continue + } + + o2 := emap2[o.Name] + if !compareAddresses(o.Addresses, o2.Addresses) { + diff = append(diff, fmt.Sprintf("%s%s\n\t%s\t%s\n\t%s\t%s", blue("Moved: "), + green(o.Name), blue(" from "), yellow(lineOfAddresses(o.Addresses)), + blue(" to "), yellow(lineOfAddresses(o2.Addresses)))) + } + } + + for _, o := range eout2 { + if _, found := handled[o.Name]; found { + continue + } + + if _, found := emap1[o.Name]; !found { + diff = append(diff, fmt.Sprintf("%s%s %s", blue("Found: "), + green(o.Name), yellow(lineOfAddresses(o.Addresses)))) + } + } + return diff +} + +func lineOfAddresses(addrs []core.AddressInfo) string { + var line string + + for i, addr := range addrs { + if i != 0 { + line = line + "," + } + line = line + addr.Address.String() + } + return line +} + +func compareAddresses(addr1, addr2 []core.AddressInfo) bool { + for _, a1 := range addr1 { + var found bool + + for _, a2 := range addr2 { + if a1.Address.Equal(a2.Address) { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func orderedEnumsAndDateRanges(enums []string, h handlers.DataHandler) ([]string, []time.Time, []time.Time) { + sort.Slice(enums, func(i, j int) bool { + var less bool + + e1, l1 := h.EnumerationDateRange(enums[i]) + e2, l2 := h.EnumerationDateRange(enums[j]) + if l2.After(l1) || e1.Before(e2) { + less = true + } + return less + }) + + var earliest, latest []time.Time + for _, enum := range enums { + e, l := h.EnumerationDateRange(enum) + + earliest = append(earliest, e) + latest = append(latest, l) + } + return enums, earliest, latest +} + +func enumContainsDomain(enum, domain string, h handlers.DataHandler) bool { + var found bool + + for _, d := range h.EnumerationDomains(enum) { + if d == domain { + found = true + break + } + } + return found +} + +func printBanner() { + rightmost := 76 + version := "Version " + amass.Version + desc := "In-depth DNS Enumeration and Network Mapping" + author := "Authored By " + amass.Author + + pad := func(num int) { + for i := 0; i < num; i++ { + fmt.Fprint(color.Error, " ") + } + } + r.Fprintln(color.Error, amass.Banner) + pad(rightmost - len(version)) + y.Fprintln(color.Error, version) + pad(rightmost - len(author)) + y.Fprintln(color.Error, author) + pad(rightmost - len(desc)) + y.Fprintf(color.Error, "%s\n\n\n", desc) +} + +// parseStrings implementation of the flag.Value interface +func (p *parseStrings) String() string { + if p == nil { + return "" + } + return strings.Join(*p, ",") +} + +func (p *parseStrings) Set(s string) error { + if s == "" { + return fmt.Errorf("String parsing failed") + } + + str := strings.Split(s, ",") + for _, s := range str { + *p = append(*p, strings.TrimSpace(s)) + } + return nil +} diff --git a/cmd/amass/main.go b/cmd/amass/main.go index 7d6692175..042ef311f 100644 --- a/cmd/amass/main.go +++ b/cmd/amass/main.go @@ -83,7 +83,7 @@ var ( // Command-line switches and provided parameters help = flag.Bool("h", false, "Show the program usage message") list = flag.Bool("list", false, "Print the names of all available data sources") - vprint = flag.Bool("version", false, "Print the version number of this amass binary") + vprint = flag.Bool("version", false, "Print the version number of this Amass binary") config = flag.String("config", "", "Path to the INI configuration file. Additional details below") unresolved = flag.Bool("include-unresolvable", false, "Output DNS names that did not resolve") ips = flag.Bool("ip", false, "Show the IP addresses for discovered names") @@ -115,7 +115,7 @@ func main() { flag.CommandLine.SetOutput(defaultBuf) flag.Usage = func() { printBanner() - g.Fprintf(color.Error, "Usage: %s [options] <-d domain>\n", path.Base(os.Args[0])) + g.Fprintf(color.Error, "Usage: %s [options] <-d domain>\n\n", path.Base(os.Args[0])) flag.PrintDefaults() g.Fprintln(color.Error, defaultBuf.String()) g.Fprintf(color.Error, "An example configuration file can be found here: \n%s\n\n", exampleConfigFileURL) @@ -465,7 +465,7 @@ func writeJSONData(f *os.File, result *core.Output) { func printBanner() { rightmost := 76 version := "Version " + amass.Version - desc := "In-Depth DNS Enumeration and Network Mapping" + desc := "In-depth DNS Enumeration and Network Mapping" author := "Authored By " + amass.Author pad := func(num int) { diff --git a/snapcraft.yaml b/snapcraft.yaml index 1e6cf4a06..e9eff2f61 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -24,6 +24,10 @@ apps: command: bin/db plugs: [home, network, removable-media] + tracker: + command: bin/tracker + plugs: [home, network, removable-media] + parts: go: @@ -91,4 +95,20 @@ parts: go install ./... mkdir $SNAPCRAFT_PART_INSTALL/bin mv $GOPATH/bin/amass.db $SNAPCRAFT_PART_INSTALL/bin/db - strip --remove-section=.comment --remove-section=.note $SNAPCRAFT_PART_INSTALL/bin/db \ No newline at end of file + strip --remove-section=.comment --remove-section=.note $SNAPCRAFT_PART_INSTALL/bin/db + + tracker: + after: [amass] + source: https://github.com/OWASP/Amass + source-type: git + plugin: go + go-importpath: github.com/OWASP/Amass + override-build: | + echo "\nStarting override-build for tracker part:" + export GOPATH=$(dirname $SNAPCRAFT_PART_INSTALL)/go + cd $GOPATH/src/github.com/OWASP/Amass + go get -u ./... + go install ./... + mkdir $SNAPCRAFT_PART_INSTALL/bin + mv $GOPATH/bin/amass.tracker $SNAPCRAFT_PART_INSTALL/bin/tracker + strip --remove-section=.comment --remove-section=.note $SNAPCRAFT_PART_INSTALL/bin/tracker \ No newline at end of file