Merge pull request #126 from caffix/various-enhancements

Various enhancements

README.md

@@ -22,7 +22,7 @@
 
 The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping. Additionally, Amass uses the IP addresses obtained during resolution to discover associated netblocks and ASNs. All the information is then used to build maps of the target networks.
 
-**Information gathering techniques used:**
+**Information Gathering Techniques Used:**
 
 * DNS: Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request)
 * Scraping: Ask, Baidu, Bing, CommonCrawl, DNSDB, DNSDumpster, DNSTable, Dogpile, Exalead, FindSubdomains, Google, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ThreatCrowd, VirusTotal, Yahoo
@@ -44,6 +44,22 @@ If your operating environment supports [Snap](https://docs.snapcraft.io/core/ins
 sudo snap install amass
 ```
 
+On Kali, follow these steps to install Snap and Amass + use AppArmor (for autoload):
+
+```bash
+sudo apt install snapd
+sudo systemctl start snapd
+sudo systemctl enable snapd
+sudo systemctl start apparmor
+sudo systemctl enable apparmor
+```
+
+Add the Snap bin directory to your PATH:
+
+```bash
+export PATH=$PATH:/snap/bin
+```
+
 Periodically, execute the following command to update all your snap packages:
 
 ```bash
@@ -121,6 +137,8 @@ This project improves thanks to all the people who contribute:
 [![Follow on Twitter](https://img.shields.io/twitter/follow/rbadguy1.svg?logo=twitter)](https://twitter.com/rbadguy1)
 [![Follow on Twitter](https://img.shields.io/twitter/follow/adam_zinger.svg?logo=twitter)](https://twitter.com/adam_zinger)
 [![Follow on Twitter](https://img.shields.io/twitter/follow/architekton1.svg?logo=twitter)](https://twitter.com/architekton1)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/danjomart.svg?logo=twitter)](https://twitter.com/danjomart)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/_b3nj4m1n__.svg?logo=twitter)](https://twitter.com/_b3nj4m1n__)
 
 ## Mentions
 

amass/amass.go

@@ -75,6 +75,7 @@ type Enumeration struct {
 	Done chan struct{}
 
 	dataSources []core.Service
+	bruteSrv    core.Service
 
 	// Pause/Resume channels for halting the enumeration
 	pause  chan struct{}
@@ -155,8 +156,8 @@ func (e *Enumeration) Start() error {
 	namesrv.RegisterGraph(e.Graph)
 	services = append(services, namesrv, NewAddressService(e.Config, e.Bus))
 	if !e.Config.Passive {
-		services = append(services, NewAlterationService(e.Config, e.Bus),
-			NewBruteForceService(e.Config, e.Bus))
+		e.bruteSrv = NewBruteForceService(e.Config, e.Bus)
+		services = append(services, NewAlterationService(e.Config, e.Bus), e.bruteSrv)
 	}
 
 	// Grab all the data sources
@@ -169,12 +170,17 @@ func (e *Enumeration) Start() error {
 
 	// Use all previously discovered names that are in scope
 	go e.submitKnownNames()
+	// Start with the first domain name provided by the configuration
+	var domainIdx int
+	e.releaseDomainName(domainIdx)
 
 	var wg sync.WaitGroup
 	wg.Add(2)
 	go e.checkForOutput(&wg)
 	go e.processOutput(&wg)
-	t := time.NewTicker(3 * time.Second)
+
+	tickSeconds := 3
+	t := time.NewTicker(time.Duration(3) * time.Second)
 	logTick := time.NewTicker(time.Minute)
 	defer logTick.Stop()
 loop:
@@ -185,9 +191,12 @@ loop:
 		case <-e.PauseChan():
 			t.Stop()
 		case <-e.ResumeChan():
-			t = time.NewTicker(3 * time.Second)
+			t = time.NewTicker(time.Duration(3) * time.Second)
 		case <-logTick.C:
-			e.processMetrics(services)
+			if !e.Config.Passive {
+				e.Config.Log.Printf("Average DNS queries performed: %d/sec, DNS names remaining: %d",
+					e.DNSQueriesPerSec(), e.DNSNamesRemaining())
+			}
 		case <-t.C:
 			done := true
 			for _, srv := range services {
@@ -198,7 +207,20 @@ loop:
 			}
 			if done {
 				close(e.Done)
+				continue
+			}
+
+			if !e.Config.Passive {
+				e.processMetrics(services)
+				psec := e.DNSQueriesPerSec()
+				// Check if it's too soon to release the next domain name
+				if psec > 0 && ((e.DNSNamesRemaining()*len(InitialQueryTypes))/psec) > tickSeconds {
+					continue
+				}
 			}
+			// Check if the next domain should be sent to data sources/brute forcing
+			domainIdx++
+			e.releaseDomainName(domainIdx)
 		}
 	}
 	t.Stop()
@@ -209,6 +231,25 @@ loop:
 	return nil
 }
 
+func (e *Enumeration) releaseDomainName(idx int) {
+	domains := e.Config.Domains()
+
+	if idx >= len(domains) {
+		return
+	}
+
+	for _, srv := range append(e.dataSources, e.bruteSrv) {
+		if srv == nil {
+			continue
+		}
+
+		srv.SendRequest(&core.Request{
+			Name:   domains[idx],
+			Domain: domains[idx],
+		})
+	}
+}
+
 func (e *Enumeration) submitKnownNames() {
 	for _, enum := range e.Graph.EnumerationList() {
 		var found bool
@@ -253,10 +294,6 @@ func (e *Enumeration) DNSNamesRemaining() int {
 }
 
 func (e *Enumeration) processMetrics(services []core.Service) {
-	if e.Config.Passive {
-		return
-	}
-
 	var total, remaining int
 	for _, srv := range services {
 		stats := srv.Stats()
@@ -265,8 +302,6 @@ func (e *Enumeration) processMetrics(services []core.Service) {
 		total += stats.DNSQueriesPerSec
 	}
 
-	e.Config.Log.Printf("Average DNS queries performed: %d/sec, DNS names remaining: %d", total, remaining)
-
 	e.metricsLock.Lock()
 	e.dnsQueriesPerSec = total
 	e.dnsNamesRemaining = remaining

amass/amass_test.go

@@ -0,0 +1,23 @@
+// Copyright 2017 Jeff Foley. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package amass
+
+import (
+	"flag"
+	"os"
+	"testing"
+)
+
+var (
+	network = flag.Bool("network", false, "Run tests that require connectivity (take more time)")
+)
+
+// TestMain will parse the test flags and setup for integration tests.
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	result := m.Run()
+
+	os.Exit(result)
+}

amass/brute.go

@@ -36,7 +36,7 @@ type BruteForceService struct {
 // NewBruteForceService returns he object initialized, but not yet started.
 func NewBruteForceService(config *core.Config, bus *core.EventBus) *BruteForceService {
 	bfs := &BruteForceService{
-		max:    utils.NewSimpleSemaphore(5000),
+		max:    utils.NewSimpleSemaphore(50000),
 		filter: utils.NewStringFilter(),
 	}
 
@@ -87,12 +87,15 @@ func (bfs *BruteForceService) processRequests() {
 }
 
 func (bfs *BruteForceService) goodRequest(req *core.Request) bool {
-	var ok bool
 	if !bfs.Config().IsDomainInScope(req.Name) {
-		return ok
+		return false
+	}
+
+	if len(req.Records) == 0 {
+		return true
 	}
 
-	bfs.SetActive()
+	var ok bool
 	for _, r := range req.Records {
 		t := uint16(r.Type)
 
@@ -178,6 +181,9 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) {
 		bfs.metrics.QueryTime(time.Now())
 		bfs.SetActive()
 	}
+	if len(answers) == 0 {
+		return
+	}
 
 	req := &core.Request{
 		Name:    name,
@@ -187,8 +193,7 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) {
 		Source:  bfs.String(),
 	}
 
-	bfs.SetActive()
-	if len(answers) == 0 || MatchesWildcard(req) {
+	if MatchesWildcard(req) {
 		return
 	}
 	bfs.Bus().Publish(core.NameResolvedTopic, req)

amass/core/request.go

@@ -66,10 +66,11 @@ type Output struct {
 
 // AddressInfo stores all network addressing info for the Output type.
 type AddressInfo struct {
-	Address     net.IP     `json:"ip"`
-	Netblock    *net.IPNet `json:"cidr"`
-	ASN         int        `json:"asn"`
-	Description string     `json:"desc"`
+	Address     net.IP `json:"ip"`
+	Netblock    *net.IPNet
+	CIDRStr     string `json:"cidr"`
+	ASN         int    `json:"asn"`
+	Description string `json:"desc"`
 }
 
 type pubReq struct {

amass/dnssrv.go

@@ -48,7 +48,7 @@ type DNSService struct {
 // NewDNSService returns he object initialized, but not yet started.
 func NewDNSService(config *core.Config, bus *core.EventBus) *DNSService {
 	ds := &DNSService{
-		max:    utils.NewSimpleSemaphore(5000),
+		max:    utils.NewSimpleSemaphore(50000),
 		filter: utils.NewStringFilter(),
 	}
 
@@ -73,10 +73,6 @@ func (ds *DNSService) OnStart() error {
 	ds.Bus().Subscribe(core.ReverseSweepTopic, ds.dnsSweep)
 	ds.Bus().Subscribe(core.NewSubdomainTopic, ds.newSubdomain)
 	go ds.processRequests()
-
-	for _, domain := range ds.Config().Domains() {
-		go ds.basicQueries(domain, domain)
-	}
 	return nil
 }
 

amass/handlers/graph.go

@@ -699,6 +699,7 @@ func (g *Graph) buildAddrInfo(addr, uuid string) *core.AddressInfo {
 	if cidr == "" {
 		return nil
 	}
+	ainfo.CIDRStr = cidr
 	_, ainfo.Netblock, _ = net.ParseCIDR(cidr)
 
 	p := cayley.StartPath(g.store, quad.String(cidr)).LabelContext(u).In(quad.String("has_prefix"))

amass/handlers/gremlin.go

@@ -785,6 +785,7 @@ func dataToOutput(path []*DataOptsParams) []*core.Output {
 		case "address":
 			addrinfo.Address = net.ParseIP(v.Address)
 		case "netblock":
+			addrinfo.CIDRStr = v.CIDR
 			_, addrinfo.Netblock, _ = net.ParseCIDR(v.CIDR)
 		case "as":
 			addrinfo.ASN = v.ASN