Merge pull request #111 from caffix/create-tracker

Initial implementation of the tracker tool

.goreleaser.yaml

@@ -38,6 +38,24 @@ builds:
       - goos: windows
         goarch: 386
 
+  - 
+    main: ./cmd/amass.viz/main.go
+    binary: amass.viz
+    goos:
+      - windows
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - 386
+    env:
+      - CGO_ENABLED=0
+    ignore:
+      - goos: darwin
+        goarch: 386
+      - goos: windows
+        goarch: 386
+
   - 
     main: ./cmd/amass.db/main.go
     binary: amass.db
@@ -55,10 +73,10 @@ builds:
         goarch: 386
       - goos: windows
         goarch: 386
-  
+
   - 
-    main: ./cmd/amass.viz/main.go
-    binary: amass.viz
+    main: ./cmd/amass.tracker/main.go
+    binary: amass.tracker
     goos:
       - windows
       - linux

README.md

@@ -22,6 +22,14 @@
 
 The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping. Additionally, Amass uses the IP addresses obtained during resolution to discover associated netblocks and ASNs. All the information is then used to build maps of the target networks.
 
+**Information gathering techniques used:**
+
+* DNS: Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request)
+* Scraping: Ask, Baidu, Bing, CommonCrawl, DNSDB, DNSDumpster, DNSTable, Dogpile, Exalead, FindSubdomains, Google, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ThreatCrowd, VirusTotal, Yahoo
+* Certificates: Active pulls (upon request), Censys, CertDB, CertSpotter, Crtsh, Entrust
+* APIs: BinaryEdge, BufferOver, CIRCL, HackerTarget, PassiveTotal, Robtex, SecurityTrails, Shodan, Twitter, Umbrella, URLScan
+* Web Archives: ArchiveIt, ArchiveToday, Arquivo, LoCArchive, OpenUKArchive, UKGovArchive, Wayback
+
 ----
 
 ## How to Install
@@ -100,8 +108,8 @@ Join our Discord server: [![Chat on Discord](https://img.shields.io/discord/4337
 
 [![Follow on Twitter](https://img.shields.io/twitter/follow/jeff_foley.svg?logo=twitter)](https://twitter.com/jeff_foley)
 
-- OWASP: [Caffix](https://www.owasp.org/index.php/User:Caffix)
-- GitHub: [@caffix](https://github.com/caffix)
+* OWASP: [Caffix](https://www.owasp.org/index.php/User:Caffix)
+* GitHub: [@caffix](https://github.com/caffix)
 
 ### Contributors
 
@@ -116,27 +124,27 @@ This project improves thanks to all the people who contribute:
 
 ## Mentions
 
-- [Pose a Threat: How Perceptual Analysis Helps Bug Hunters](https://www.bishopfox.com/news/2018/12/appsec-california-pose-a-threat-how-perpetual-analysis-helps-bug-hunters/)
-- [A penetration tester’s guide to subdomain enumeration](https://blog.appsecco.com/a-penetration-testers-guide-to-sub-domain-enumeration-7d842d5570f6)
-- [Abusing access control on a large online e-commerce site to register as supplier](https://medium.com/@fbotes2/governit-754becf85cbc)
-- [Black Hat Training, Making the Cloud Rain Shells!: Discovery and Recon](https://www.blackhat.com/eu-18/training/schedule/index.html#aws--azure-exploitation-making-the-cloud-rain-shells-11060)
-- [Subdomains Enumeration Cheat Sheet](https://pentester.land/cheatsheets/2018/11/14/subdomains-enumeration-cheatsheet.html)
-- [Search subdomains and build graphs of network structure with Amass](https://miloserdov.org/?p=2309)
-- [Getting started in Bug Bounty](https://medium.com/@ehsahil/getting-started-in-bug-bounty-7052da28445a)
-- [Source code disclosure via exposed .git folder](https://pentester.land/tutorials/2018/10/25/source-code-disclosure-via-exposed-git-folder.html)
-- [Amass, the best application to search for subdomains](https://www.h1rd.com/hacking/amass-para-buscar-subdominios)
-- [Subdomain Takeover: Finding Candidates](https://0xpatrik.com/subdomain-takeover-candidates/)
-- [Paul's Security Weekly #564: Technical Segment - Bug Bounty Hunting](https://wiki.securityweekly.com/Episode564)
-- [The Bug Hunters Methodology v3(ish)](https://www.youtube.com/watch?v=Qw1nNPiH_Go)
-- [Doing Recon the Correct Way](https://enciphers.com/doing-recon-the-correct-way/)
-- [Discovering subdomains](https://www.sjoerdlangkemper.nl/2018/06/20/discovering-subdomains/)
-- [Best Hacking Tools List for Hackers & Security Professionals 2018](http://kalilinuxtutorials.com/best-hacking-tools-list/amp/)
-- [Amass - Subdomain Enumeration Tool](https://hydrasky.com/network-security/kali-tools/amass-subdomain-enumeration-tool/)
-- [Subdomain enumeration](http://10degres.net/subdomain-enumeration/)
-- [Asset Discovery: Doing Reconnaissance the Hard Way](https://0xpatrik.com/asset-discovery/)
-- [Project Sonar: An Underrated Source of Internet-wide Data](https://0xpatrik.com/project-sonar-guide/)
-- [Go is for everyone](https://changelog.com/gotime/71)
-- [Top Five Ways the Red Team breached the External Perimeter](https://medium.com/@adam.toscher/top-five-ways-the-red-team-breached-the-external-perimeter-262f99dc9d17)
+* [Pose a Threat: How Perceptual Analysis Helps Bug Hunters](https://www.bishopfox.com/news/2018/12/appsec-california-pose-a-threat-how-perpetual-analysis-helps-bug-hunters/)
+* [A penetration tester’s guide to subdomain enumeration](https://blog.appsecco.com/a-penetration-testers-guide-to-sub-domain-enumeration-7d842d5570f6)
+* [Abusing access control on a large online e-commerce site to register as supplier](https://medium.com/@fbotes2/governit-754becf85cbc)
+* [Black Hat Training, Making the Cloud Rain Shells!: Discovery and Recon](https://www.blackhat.com/eu-18/training/schedule/index.html#aws--azure-exploitation-making-the-cloud-rain-shells-11060)
+* [Subdomains Enumeration Cheat Sheet](https://pentester.land/cheatsheets/2018/11/14/subdomains-enumeration-cheatsheet.html)
+* [Search subdomains and build graphs of network structure with Amass](https://miloserdov.org/?p=2309)
+* [Getting started in Bug Bounty](https://medium.com/@ehsahil/getting-started-in-bug-bounty-7052da28445a)
+* [Source code disclosure via exposed .git folder](https://pentester.land/tutorials/2018/10/25/source-code-disclosure-via-exposed-git-folder.html)
+* [Amass, the best application to search for subdomains](https://www.h1rd.com/hacking/amass-para-buscar-subdominios)
+* [Subdomain Takeover: Finding Candidates](https://0xpatrik.com/subdomain-takeover-candidates/)
+* [Paul's Security Weekly #564: Technical Segment - Bug Bounty Hunting](https://wiki.securityweekly.com/Episode564)
+* [The Bug Hunters Methodology v3(ish)](https://www.youtube.com/watch?v=Qw1nNPiH_Go)
+* [Doing Recon the Correct Way](https://enciphers.com/doing-recon-the-correct-way/)
+* [Discovering subdomains](https://www.sjoerdlangkemper.nl/2018/06/20/discovering-subdomains/)
+* [Best Hacking Tools List for Hackers & Security Professionals 2018](http://kalilinuxtutorials.com/best-hacking-tools-list/amp/)
+* [Amass - Subdomain Enumeration Tool](https://hydrasky.com/network-security/kali-tools/amass-subdomain-enumeration-tool/)
+* [Subdomain enumeration](http://10degres.net/subdomain-enumeration/)
+* [Asset Discovery: Doing Reconnaissance the Hard Way](https://0xpatrik.com/asset-discovery/)
+* [Project Sonar: An Underrated Source of Internet-wide Data](https://0xpatrik.com/project-sonar-guide/)
+* [Go is for everyone](https://changelog.com/gotime/71)
+* [Top Five Ways the Red Team breached the External Perimeter](https://medium.com/@adam.toscher/top-five-ways-the-red-team-breached-the-external-perimeter-262f99dc9d17)
 
 ## Stargazers over Time
 

amass/amass.go

@@ -151,6 +151,9 @@ func (e *Enumeration) Start() error {
 		}
 	}
 
+	// Use all previously discovered names that are in scope
+	go e.submitKnownNames()
+
 	var wg sync.WaitGroup
 	wg.Add(2)
 	go e.checkForOutput(&wg)
@@ -190,6 +193,33 @@ loop:
 	return nil
 }
 
+func (e *Enumeration) submitKnownNames() {
+	for _, enum := range e.Graph.EnumerationList() {
+		var found bool
+
+		for _, domain := range e.Graph.EnumerationDomains(enum) {
+			if e.Config.IsDomainInScope(domain) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+
+		for _, o := range e.Graph.GetOutput(enum, true) {
+			if e.Config.IsDomainInScope(o.Name) {
+				e.Bus.Publish(core.NewNameTopic, &core.Request{
+					Name:   o.Name,
+					Domain: o.Domain,
+					Tag:    o.Tag,
+					Source: o.Source,
+				})
+			}
+		}
+	}
+}
+
 // DNSQueriesPerSec returns the number of DNS queries the enumeration has performed per second.
 func (e *Enumeration) DNSQueriesPerSec() int {
 	e.metricsLock.RLock()
@@ -279,7 +309,7 @@ loop:
 		case <-e.Done:
 			break loop
 		case <-t.C:
-			out := e.Graph.GetUnreadOutput(e.Config.UUID.String())
+			out := e.Graph.GetOutput(e.Config.UUID.String(), false)
 			for _, o := range out {
 				if time.Now().Add(10 * time.Second).After(o.Timestamp) {
 					e.Graph.MarkAsRead(&handlers.DataOptsParams{
@@ -296,7 +326,7 @@ loop:
 		}
 	}
 	// Handle all remaining pieces of output
-	out := e.Graph.GetUnreadOutput(e.Config.UUID.String())
+	out := e.Graph.GetOutput(e.Config.UUID.String(), false)
 	for _, o := range out {
 		if !e.filter.Duplicate(o.Name) {
 			e.Graph.MarkAsRead(&handlers.DataOptsParams{

amass/brute.go

@@ -36,7 +36,7 @@ type BruteForceService struct {
 // NewBruteForceService returns he object initialized, but not yet started.
 func NewBruteForceService(config *core.Config, bus *core.EventBus) *BruteForceService {
 	bfs := &BruteForceService{
-		max:    utils.NewSimpleSemaphore(10000),
+		max:    utils.NewSimpleSemaphore(5000),
 		filter: utils.NewStringFilter(),
 	}
 

amass/dnssrv.go

@@ -48,7 +48,7 @@ type DNSService struct {
 // NewDNSService returns he object initialized, but not yet started.
 func NewDNSService(config *core.Config, bus *core.EventBus) *DNSService {
 	ds := &DNSService{
-		max:    utils.NewSimpleSemaphore(10000),
+		max:    utils.NewSimpleSemaphore(5000),
 		filter: utils.NewStringFilter(),
 	}
 

amass/handlers/dataopts.go

@@ -6,6 +6,7 @@ package handlers
 import (
 	"encoding/json"
 	"io"
+	"time"
 
 	"github.com/OWASP/Amass/amass/core"
 )
@@ -35,6 +36,26 @@ func (d *DataOptsHandler) Insert(data *DataOptsParams) error {
 	return d.Enc.Encode(data)
 }
 
+// EnumerationList returns a list of enumeration IDs found in the data.
+func (d *DataOptsHandler) EnumerationList() []string {
+	return []string{}
+}
+
+// EnumerationDomains returns the domains that were involved in the provided enumeration.
+func (d *DataOptsHandler) EnumerationDomains(uuid string) []string {
+	return []string{}
+}
+
+// EnumerationDateRange returns the date range associated with the provided enumeration UUID.
+func (d *DataOptsHandler) EnumerationDateRange(uuid string) (time.Time, time.Time) {
+	return time.Now(), time.Now()
+}
+
+// GetOutput implements the Amass DataHandler interface.
+func (d *DataOptsHandler) GetOutput(uuid string, marked bool) []*core.Output {
+	return nil
+}
+
 // MarkAsRead implements the Amass DataHandler interface.
 func (d *DataOptsHandler) MarkAsRead(data *DataOptsParams) error {
 	return nil
@@ -44,8 +65,3 @@ func (d *DataOptsHandler) MarkAsRead(data *DataOptsParams) error {
 func (d *DataOptsHandler) IsCNAMENode(data *DataOptsParams) bool {
 	return false
 }
-
-// GetUnreadOutput implements the Amass DataHandler interface.
-func (d *DataOptsHandler) GetUnreadOutput(uuid string) []*core.Output {
-	return nil
-}