Merge pull request #4 from projectdiscovery/feature-hosts-file

adding hosts file support

fastdialer/dialer.go

@@ -37,6 +37,12 @@ func NewDialer(options Options) (*Dialer, error) {
 		DualStack: true,
 	}
 
+	// load hardcoded values from host file
+	if options.HostsFile {
+		// nolint:errcheck // if they cannot be loaded it's not a hard failure
+		loadHostsFile(hm)
+	}
+
 	return &Dialer{dnsclient: dnsclient, hm: hm, dialerHistory: dialerHistory, dialer: dialer}, nil
 }
 

fastdialer/hostsfile.go

@@ -0,0 +1,84 @@
+package fastdialer
+
+import (
+	"bufio"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/dimchansky/utfbom"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	retryabledns "github.com/projectdiscovery/retryabledns"
+)
+
+func loadHostsFile(hm *hybrid.HybridMap) error {
+	osHostsFilePath := os.ExpandEnv(filepath.FromSlash(HostsFilePath))
+
+	if env, isset := os.LookupEnv("HOSTS_PATH"); isset && len(env) > 0 {
+		osHostsFilePath = os.ExpandEnv(filepath.FromSlash(env))
+	}
+
+	file, err := os.Open(osHostsFilePath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(utfbom.SkipOnly(file))
+	for scanner.Scan() {
+		ip, hosts := HandleLine(scanner.Text())
+		if ip == "" || len(hosts) == 0 {
+			continue
+		}
+		for _, host := range hosts {
+			dnsdata := retryabledns.DNSData{Host: host, A: []string{ip}}
+			dnsdataBytes, _ := dnsdata.Marshal()
+			// nolint:errcheck // if they cannot be cached it's not a hard failure
+			hm.Set(host, dnsdataBytes)
+		}
+	}
+
+	return nil
+}
+
+const commentChar string = "#"
+
+// HandleLine a hosts file line
+func HandleLine(raw string) (ip string, hosts []string) {
+	// ignore comment
+	if IsComment(raw) {
+		return
+	}
+
+	// trim comment
+	if HasComment(raw) {
+		commentSplit := strings.Split(raw, commentChar)
+		raw = commentSplit[0]
+	}
+
+	fields := strings.Fields(raw)
+	if len(fields) == 0 {
+		return
+	}
+
+	// not a valid ip
+	ip = fields[0]
+	if net.ParseIP(ip) == nil {
+		return
+	}
+
+	hosts = fields[1:]
+
+	return
+}
+
+// IsComment check if the file is a comment
+func IsComment(raw string) bool {
+	return strings.HasPrefix(strings.TrimSpace(raw), commentChar)
+}
+
+// HasComment check if the line has a comment
+func HasComment(raw string) bool {
+	return strings.Contains(raw, commentChar)
+}

fastdialer/hostsfile_unix.go

@@ -0,0 +1,6 @@
+// +build !windows
+
+package fastdialer
+
+// HostsFilePath in unix file os
+const HostsFilePath = "/etc/hosts"

fastdialer/hostsfile_windows.go

@@ -0,0 +1,4 @@
+// +build windows
+package fastdialer
+
+const HostsFilePath = "${SystemRoot}/System32/drivers/etc/hosts"

fastdialer/options.go

@@ -11,10 +11,12 @@ var DefaultResolvers = []string{
 type Options struct {
 	BaseResolvers []string
 	MaxRetries    int
+	HostsFile     bool
 }
 
 // DefaultOptions of the cache
 var DefaultOptions = Options{
 	BaseResolvers: DefaultResolvers,
 	MaxRetries:    5,
+	HostsFile:     true,
 }

go.mod

@@ -3,6 +3,7 @@ module github.com/projectdiscovery/fastdialer
 go 1.14
 
 require (
+	github.com/dimchansky/utfbom v1.1.1
 	github.com/projectdiscovery/hmap v0.0.1
 	github.com/projectdiscovery/retryabledns v1.0.5
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=