Skip to content

Commit

Permalink
2021 refactor of the enumeration
Browse files Browse the repository at this point in the history
  • Loading branch information
caffix committed Jan 12, 2021
1 parent ad2ed9c commit 4432dbd
Show file tree
Hide file tree
Showing 118 changed files with 4,327 additions and 6,105 deletions.
2 changes: 1 addition & 1 deletion cmd/amass/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"github.com/OWASP/Amass/v3/graph"
"github.com/OWASP/Amass/v3/net"
"github.com/OWASP/Amass/v3/requests"
"github.com/OWASP/Amass/v3/stringset"
"github.com/caffix/stringset"
"github.com/fatih/color"
)

Expand Down
119 changes: 82 additions & 37 deletions cmd/amass/dns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,18 @@ import (
"fmt"
"math/rand"
"os"
"os/signal"
"strings"
"syscall"
"time"

"github.com/OWASP/Amass/v3/config"
"github.com/OWASP/Amass/v3/eventbus"
"github.com/OWASP/Amass/v3/format"
"github.com/OWASP/Amass/v3/requests"
"github.com/OWASP/Amass/v3/resolvers"
"github.com/OWASP/Amass/v3/stringset"
"github.com/OWASP/Amass/v3/systems"
"github.com/caffix/eventbus"
"github.com/caffix/stringset"
"github.com/fatih/color"
"github.com/miekg/dns"
)
Expand Down Expand Up @@ -90,7 +92,7 @@ func defineDNSFilepathFlags(dnsFlags *flag.FlagSet, args *dnsArgs) {
}

func runDNSCommand(clArgs []string) {
args := dnsArgs{
args := &dnsArgs{
Blacklist: stringset.New(),
Domains: stringset.New(),
Names: stringset.New(),
Expand All @@ -105,9 +107,9 @@ func runDNSCommand(clArgs []string) {

dnsCommand.BoolVar(&help1, "h", false, "Show the program usage message")
dnsCommand.BoolVar(&help2, "help", false, "Show the program usage message")
defineDNSArgumentFlags(dnsCommand, &args)
defineDNSOptionFlags(dnsCommand, &args)
defineDNSFilepathFlags(dnsCommand, &args)
defineDNSArgumentFlags(dnsCommand, args)
defineDNSOptionFlags(dnsCommand, args)
defineDNSFilepathFlags(dnsCommand, args)

if len(clArgs) < 1 {
commandUsage(dnsUsageMsg, dnsCommand, dnsBuf)
Expand All @@ -123,7 +125,7 @@ func runDNSCommand(clArgs []string) {
return
}

if err := processDNSInputFiles(&args); err != nil {
if err := processDNSInputFiles(args); err != nil {
fmt.Fprintf(color.Error, "%v\n", err)
os.Exit(1)
}
Expand Down Expand Up @@ -157,44 +159,43 @@ func runDNSCommand(clArgs []string) {
os.Exit(1)
}

performResolutions(cfg, sys)
performResolutions(cfg, args, sys)
}

func performResolutions(cfg *config.Config, sys systems.System) {
func performResolutions(cfg *config.Config, args *dnsArgs, sys systems.System) {
done := make(chan struct{})
active := make(chan struct{}, 1000000)
bus := eventbus.NewEventBus()
answers := make(chan *requests.DNSRequest, 100000)

// Setup the context used throughout the resolutions
ctx, cancel := context.WithCancel(context.Background())
ctx = context.WithValue(ctx, requests.ContextEventBus, bus)

if cfg.Timeout > 0 {
time.AfterFunc(time.Duration(cfg.Timeout)*time.Minute, func() {
close(done)
})
var ctx context.Context
var cancel context.CancelFunc
if args.Timeout == 0 {
ctx, cancel = context.WithCancel(context.Background())
} else {
ctx, cancel = context.WithTimeout(context.Background(), time.Duration(args.Timeout)*time.Minute)
}
defer cancel()
ctx = context.WithValue(ctx, requests.ContextEventBus, bus)

activeFunc := func(s string) { active <- struct{}{} }
resolvFunc := func(t time.Time, rcode int) { active <- struct{}{} }
bus.Subscribe(requests.SetActiveTopic, activeFunc)
defer bus.Unsubscribe(requests.SetActiveTopic, activeFunc)
bus.Subscribe(requests.ResolveCompleted, resolvFunc)
defer bus.Unsubscribe(requests.ResolveCompleted, resolvFunc)
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)

go func() {
loop:
for _, name := range cfg.ProvidedNames {
select {
case <-done:
cancel()
return
break loop
case <-quit:
break loop
default:
if sys.PerformDNSQuery() == nil {
go processDNSRequest(ctx, &requests.DNSRequest{Name: name}, cfg, sys, answers)
}
go processDNSRequest(ctx, &requests.DNSRequest{Name: name}, cfg, sys, answers)
}
}
cancel()
close(done)
}()

processDNSAnswers(cfg, active, answers, done)
Expand All @@ -208,31 +209,52 @@ func processDNSRequest(ctx context.Context, req *requests.DNSRequest,
return
}

req.Domain = sys.Pool().SubdomainToDomain(req.Name)
req.Domain = resolvers.FirstProperSubdomain(ctx, sys.Pool(), req.Name, resolvers.PriorityHigh)
if req.Domain == "" {
c <- nil
return
}

if cfg.Blacklisted(req.Name) || sys.Pool().GetWildcardType(ctx, req) == resolvers.WildcardTypeDynamic {
msg := resolvers.QueryMsg(req.Name, dns.TypeNone)
if cfg.Blacklisted(req.Name) || sys.Pool().WildcardType(ctx, msg, req.Domain) == resolvers.WildcardTypeDynamic {
c <- nil
return
}

var answers []requests.DNSAnswer
for _, t := range cfg.RecordTypes {
a, err := sys.Pool().Resolve(ctx, req.Name, t, resolvers.PriorityLow, resolvers.RetryPolicy)
qtype := nameToType(t)
msg := resolvers.QueryMsg(req.Name, qtype)
resp, err := sys.Pool().Query(ctx, msg, resolvers.PriorityLow, resolvers.RetryPolicy)
if err == nil {
answers = append(answers, a...)
}
ans := resolvers.ExtractAnswers(resp)
if len(ans) == 0 {
continue
}

if t == "CNAME" && len(answers) > 0 {
rr := resolvers.AnswersByType(ans, qtype)
if len(rr) == 0 {
continue
}

for _, a := range rr {
answers = append(answers, requests.DNSAnswer{
Name: a.Name,
Type: int(a.Type),
Data: a.Data,
})
}
}
if t == "CNAME" && len(resp.Answer) > 0 {
break
}
if sys.Pool().WildcardType(ctx, msg, req.Domain) != resolvers.WildcardTypeNone {
return
}
}
req.Records = answers

if len(req.Records) == 0 || sys.Pool().MatchesWildcard(ctx, req) {
if len(req.Records) == 0 {
c <- nil
return
}
Expand Down Expand Up @@ -343,9 +365,6 @@ func (d dnsArgs) OverrideConfig(conf *config.Config) error {
if len(d.Blacklist) > 0 {
conf.Blacklist = d.Blacklist.Slice()
}
if d.Timeout > 0 {
conf.Timeout = d.Timeout
}
if d.Options.Verbose {
conf.Verbose = true
}
Expand Down Expand Up @@ -408,3 +427,29 @@ func typeToName(qtype uint16) string {

return name
}

func nameToType(t string) uint16 {
switch t {
case "CNAME":
return dns.TypeCNAME
case "A":
return dns.TypeA
case "AAAA":
return dns.TypeAAAA
case "PTR":
return dns.TypePTR
case "NS":
return dns.TypeNS
case "MX":
return dns.TypeMX
case "TXT":
return dns.TypeTXT
case "SOA":
return dns.TypeSOA
case "SPF":
return dns.TypeSPF
case "SRV":
return dns.TypeSRV
}
return dns.TypeNone
}
75 changes: 35 additions & 40 deletions cmd/amass/enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
Expand All @@ -29,8 +30,8 @@ import (
"github.com/OWASP/Amass/v3/format"
"github.com/OWASP/Amass/v3/requests"
"github.com/OWASP/Amass/v3/stringfilter"
"github.com/OWASP/Amass/v3/stringset"
"github.com/OWASP/Amass/v3/systems"
"github.com/caffix/stringset"
"github.com/fatih/color"
)

Expand Down Expand Up @@ -176,7 +177,7 @@ func runEnumCommand(clArgs []string) {
os.Exit(1)
}
defer sys.Shutdown()
sys.SetDataSources(datasrcs.GetAllSources(sys, true))
sys.SetDataSources(datasrcs.GetAllSources(sys))
// Expand data source category names into the associated source names
cfg.SourceFilter.Sources = expandCategoryNames(cfg.SourceFilter.Sources, generateCategoryMap(sys))

Expand Down Expand Up @@ -214,9 +215,30 @@ func runEnumCommand(clArgs []string) {
wg.Add(1)
go processOutput(e, outChans, done, &wg)

go signalHandler(e)
var ctx context.Context
var cancel context.CancelFunc
if args.Timeout == 0 {
ctx, cancel = context.WithCancel(context.Background())
} else {
ctx, cancel = context.WithTimeout(context.Background(), time.Duration(args.Timeout)*time.Minute)
}
defer cancel()

// Monitor for cancellation by the user
go func() {
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)

select {
case <-quit:
cancel()
case <-done:
case <-ctx.Done():
}
}()

// Start the enumeration process
if err := e.Start(); err != nil {
if err := e.Start(ctx); err != nil {
r.Println(err)
os.Exit(1)
}
Expand Down Expand Up @@ -271,7 +293,6 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) {
commandUsage(enumUsageMsg, enumCommand, enumBuf)
return nil, &args
}

if err := enumCommand.Parse(clArgs); err != nil {
r.Fprintf(color.Error, "%v\n", err)
os.Exit(1)
Expand Down Expand Up @@ -327,7 +348,6 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) {
r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
os.Exit(1)
}

// Override configuration file settings with command-line arguments
if err := cfg.UpdateConfig(args); err != nil {
r.Fprintf(color.Error, "Configuration error: %v\n", err)
Expand Down Expand Up @@ -445,7 +465,6 @@ func saveJSONOutput(e *enum.Enumeration, args *enumArgs, output chan *requests.O
if args.Filepaths.AllFilePrefix != "" {
jsonfile = args.Filepaths.AllFilePrefix + ".json"
}

if jsonfile == "" {
return
}
Expand Down Expand Up @@ -516,26 +535,10 @@ loop:
}
}

// If the user interrupts the program, print the summary information
func signalHandler(e *enum.Enumeration) {
quit := make(chan os.Signal, 1)

signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
<-quit
// Signal the enumeration to finish
e.Done()
// Wait for output operations to complete
time.Sleep(30 * time.Second)
os.Exit(1)
}

func writeLogsAndMessages(logs *io.PipeReader, logfile string, verbose bool) {
wildcard := regexp.MustCompile("DNS wildcard")
avg := regexp.MustCompile("Average DNS queries")
rScore := regexp.MustCompile("Resolver .* has a low score")
alterations := regexp.MustCompile("queries for altered names")
brute := regexp.MustCompile("queries for brute forcing")
sanity := regexp.MustCompile("SanityChecks")
queries := regexp.MustCompile("Querying")

var filePtr *os.File
Expand Down Expand Up @@ -577,24 +580,12 @@ func writeLogsAndMessages(logs *io.PipeReader, logfile string, verbose bool) {
if rScore.FindString(line) != "" {
fgR.Fprintln(color.Error, line)
}
// Let the user know when brute forcing has started
if brute.FindString(line) != "" {
fgY.Fprintln(color.Error, line)
}
// Let the user know when name alterations have started
if alterations.FindString(line) != "" {
fgY.Fprintln(color.Error, line)
}
// Check if DNS resolvers have failed the sanity checks
if verbose && sanity.FindString(line) != "" {
fgR.Fprintln(color.Error, line)
}
// Check for Amass DNS wildcard messages
if verbose && wildcard.FindString(line) != "" {
fgR.Fprintln(color.Error, line)
}
// Let the user know when data sources are being queried
if queries.FindString(line) != "" {
if verbose && queries.FindString(line) != "" {
fgY.Fprintln(color.Error, line)
}
}
Expand Down Expand Up @@ -729,9 +720,6 @@ func (e enumArgs) OverrideConfig(conf *config.Config) error {
if e.Blacklist.Len() > 0 {
conf.Blacklist = e.Blacklist.Slice()
}
if e.Timeout > 0 {
conf.Timeout = e.Timeout
}
if e.Options.Verbose {
conf.Verbose = true
}
Expand All @@ -755,8 +743,15 @@ func (e enumArgs) OverrideConfig(conf *config.Config) error {
e.Included.Insert(requests.BRUTE)
}
conf.SourceFilter.Sources = e.Included.Slice()
} else if len(e.Excluded) > 0 {
} else if len(e.Excluded) > 0 || conf.Alterations || conf.BruteForcing {
conf.SourceFilter.Include = false
// Check if brute forcing and alterations should be added
if conf.Alterations {
e.Included.Insert(requests.ALT)
}
if conf.BruteForcing {
e.Included.Insert(requests.BRUTE)
}
conf.SourceFilter.Sources = e.Excluded.Slice()
}
// Attempt to add the provided domains to the configuration
Expand Down
Loading

0 comments on commit 4432dbd

Please sign in to comment.