Merge: + DNS: use rules from /etc/hosts

- fix filtering logic: don't do DNS response check for Rewrite rules

Close #1478

Squashed commit of the following:

commit 1206b94
Merge: c462577 5fe9847
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 15:00:25 2020 +0300

    Merge remote-tracking branch 'origin/master' into 1478-auto-records

commit c462577
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 14:33:17 2020 +0300

    minor

commit 7e824ba
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 14:29:54 2020 +0300

    more tests

commit a22b621
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 14:09:52 2020 +0300

    rename, move

commit 9e5ed49
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 15:33:27 2020 +0300

    fix logic - don't do DNS response check for Rewrite rules

commit 6cfabc0
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 11:35:07 2020 +0300

    minor

commit 4540aed
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 11:03:24 2020 +0300

    fix

commit 9ddddf7
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 10:49:13 2020 +0300

    fix

commit c5f8ef7
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 10:37:26 2020 +0300

    fix

commit f4be009
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Mon Mar 16 20:13:00 2020 +0300

    + auto DNS records from /etc/hosts

dnsfilter/dnsfilter.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/AdguardTeam/AdGuardHome/util"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/cache"
 	"github.com/AdguardTeam/golibs/log"
@@ -53,6 +54,9 @@ type Config struct {
 	// Per-client settings can override this configuration.
 	BlockedServices []string `yaml:"blocked_services"`
 
+	// IP-hostname pairs taken from system configuration (e.g. /etc/hosts) files
+	AutoHosts *util.AutoHosts `yaml:"-"`
+
 	// Called when the configuration is changed by HTTP request
 	ConfigModified func() `yaml:"-"`
 
@@ -139,6 +143,9 @@ const (
 
 	// ReasonRewrite - rewrite rule was applied
 	ReasonRewrite
+
+	// RewriteEtcHosts - rewrite by /etc/hosts rule
+	RewriteEtcHosts
 )
 
 var reasonNames = []string{
@@ -154,6 +161,7 @@ var reasonNames = []string{
 	"FilteredBlockedService",
 
 	"Rewrite",
+	"RewriteEtcHosts",
 }
 
 func (r Reason) String() string {
@@ -303,6 +311,15 @@ func (d *Dnsfilter) CheckHost(host string, qtype uint16, setts *RequestFiltering
 		return result, nil
 	}
 
+	if d.Config.AutoHosts != nil {
+		ips := d.Config.AutoHosts.Process(host)
+		if ips != nil {
+			result.Reason = RewriteEtcHosts
+			result.IPList = ips
+			return result, nil
+		}
+	}
+
 	// try filter lists first
 	if setts.FilteringEnabled {
 		result, err = d.matchHost(host, qtype, setts.ClientTags)

dnsfilter/dnsfilter_test.go

@@ -3,6 +3,7 @@ package dnsfilter
 import (
 	"fmt"
 	"net"
+	"os"
 	"path"
 	"runtime"
 	"testing"
@@ -621,6 +622,13 @@ func TestRewrites(t *testing.T) {
 	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
 }
 
+func prepareTestDir() string {
+	const dir = "./agh-test"
+	_ = os.RemoveAll(dir)
+	_ = os.MkdirAll(dir, 0755)
+	return dir
+}
+
 // BENCHMARKS
 
 func BenchmarkSafeBrowsing(b *testing.B) {

dnsforward/dnsforward.go

@@ -665,7 +665,11 @@ func processFilteringAfterResponse(ctx *dnsContext) int {
 	res := ctx.result
 	var err error
 
-	if res.Reason == dnsfilter.ReasonRewrite && len(res.CanonName) != 0 {
+	switch res.Reason {
+	case dnsfilter.ReasonRewrite:
+		if len(res.CanonName) == 0 {
+			break
+		}
 		d.Req.Question[0] = ctx.origQuestion
 		d.Res.Question[0] = ctx.origQuestion
 
@@ -676,7 +680,14 @@ func processFilteringAfterResponse(ctx *dnsContext) int {
 			d.Res.Answer = answer
 		}
 
-	} else if res.Reason != dnsfilter.NotFilteredWhiteList && ctx.protectionEnabled {
+	case dnsfilter.RewriteEtcHosts:
+	case dnsfilter.NotFilteredWhiteList:
+		// nothing
+
+	default:
+		if !ctx.protectionEnabled {
+			break
+		}
 		origResp2 := d.Res
 		ctx.result, err = s.filterDNSResponse(ctx)
 		if err != nil {
@@ -845,7 +856,8 @@ func (s *Server) filterDNSRequest(ctx *dnsContext) (*dnsfilter.Result, error) {
 		// log.Tracef("Host %s is filtered, reason - '%s', matched rule: '%s'", host, res.Reason, res.Rule)
 		d.Res = s.genDNSFilterMessage(d, &res)
 
-	} else if res.Reason == dnsfilter.ReasonRewrite && len(res.IPList) != 0 {
+	} else if (res.Reason == dnsfilter.ReasonRewrite || res.Reason == dnsfilter.RewriteEtcHosts) &&
+		len(res.IPList) != 0 {
 		resp := s.makeResponse(req)
 
 		name := host

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/AdguardTeam/urlfilter v0.9.1
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/etcd-io/bbolt v1.3.3
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-test/deep v1.0.4 // indirect
 	github.com/gobuffalo/packr v1.19.0
 	github.com/joomcode/errorx v1.0.0

go.sum

@@ -40,6 +40,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/etcd-io/bbolt v1.3.3 h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@@ -181,6 +183,7 @@ golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed h1:5TJcLJn2a55mJjzYk0yOoqN8X1OdvBDUnaZaKKyQtkY=
 golang.org/x/sys v0.0.0-20191002091554-b397fe3ad8ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 h1:JA8d3MPx/IToSyXZG/RhwYEtfrKO1Fxrqe8KrkiLXKM=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=

home/clients.go

@@ -3,9 +3,7 @@ package home
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"net"
-	"os"
 	"os/exec"
 	"runtime"
 	"sort"
@@ -16,6 +14,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/dhcpd"
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
 	"github.com/AdguardTeam/AdGuardHome/dnsforward"
+	"github.com/AdguardTeam/AdGuardHome/util"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/utils"
@@ -79,12 +78,14 @@ type clientsContainer struct {
 	// dhcpServer is used for looking up clients IP addresses by MAC addresses
 	dhcpServer *dhcpd.Server
 
+	autoHosts *util.AutoHosts // get entries from system hosts-files
+
 	testing bool // if TRUE, this object is used for internal tests
 }
 
 // Init initializes clients container
 // Note: this function must be called only once
-func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.Server) {
+func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.Server, autoHosts *util.AutoHosts) {
 	if clients.list != nil {
 		log.Fatal("clients.list != nil")
 	}
@@ -98,11 +99,13 @@ func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.
 	}
 
 	clients.dhcpServer = dhcpServer
+	clients.autoHosts = autoHosts
 	clients.addFromConfig(objects)
 
 	if !clients.testing {
 		clients.addFromDHCP()
 		clients.dhcpServer.SetOnLeaseChanged(clients.onDHCPLeaseChanged)
+		clients.autoHosts.SetOnChanged(clients.onHostsChanged)
 	}
 }
 
@@ -120,7 +123,6 @@ func (clients *clientsContainer) Start() {
 
 // Reload - reload auto-clients
 func (clients *clientsContainer) Reload() {
-	clients.addFromHostsFile()
 	clients.addFromSystemARP()
 }
 
@@ -225,6 +227,10 @@ func (clients *clientsContainer) onDHCPLeaseChanged(flags int) {
 	}
 }
 
+func (clients *clientsContainer) onHostsChanged() {
+	clients.addFromHostsFile()
+}
+
 // Exists checks if client with this IP already exists
 func (clients *clientsContainer) Exists(ip string, source clientSource) bool {
 	clients.lock.Lock()
@@ -605,46 +611,28 @@ func (clients *clientsContainer) rmHosts(source clientSource) int {
 	return n
 }
 
-// Parse system 'hosts' file and fill clients array
+// Fill clients array from system hosts-file
 func (clients *clientsContainer) addFromHostsFile() {
-	hostsFn := "/etc/hosts"
-	if runtime.GOOS == "windows" {
-		hostsFn = os.ExpandEnv("$SystemRoot\\system32\\drivers\\etc\\hosts")
-	}
-
-	d, e := ioutil.ReadFile(hostsFn)
-	if e != nil {
-		log.Info("Can't read file %s: %v", hostsFn, e)
-		return
-	}
+	hosts := clients.autoHosts.List()
 
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 	_ = clients.rmHosts(ClientSourceHostsFile)
 
-	lines := strings.Split(string(d), "\n")
 	n := 0
-	for _, ln := range lines {
-		ln = strings.TrimSpace(ln)
-		if len(ln) == 0 || ln[0] == '#' {
-			continue
-		}
-
-		fields := strings.Fields(ln)
-		if len(fields) < 2 {
-			continue
-		}
-
-		ok, e := clients.addHost(fields[0], fields[1], ClientSourceHostsFile)
-		if e != nil {
-			log.Tracef("%s", e)
-		}
-		if ok {
-			n++
+	for ip, names := range hosts {
+		for _, name := range names {
+			ok, err := clients.addHost(ip, name.String(), ClientSourceHostsFile)
+			if err != nil {
+				log.Debug("Clients: %s", err)
+			}
+			if ok {
+				n++
+			}
 		}
 	}
 
-	log.Debug("Clients: added %d client aliases from %s", n, hostsFn)
+	log.Debug("Clients: added %d client aliases from system hosts-file", n)
 }
 
 // Add IP -> Host pairs from the system's `arp -a` command output

home/clients_test.go

@@ -18,7 +18,7 @@ func TestClients(t *testing.T) {
 	clients := clientsContainer{}
 	clients.testing = true
 
-	clients.Init(nil, nil)
+	clients.Init(nil, nil, nil)
 
 	// add
 	c = Client{
@@ -156,7 +156,7 @@ func TestClientsWhois(t *testing.T) {
 	var c Client
 	clients := clientsContainer{}
 	clients.testing = true
-	clients.Init(nil, nil)
+	clients.Init(nil, nil, nil)
 
 	whois := [][]string{{"orgname", "orgname-val"}, {"country", "country-val"}}
 	// set whois info on new client
@@ -183,7 +183,7 @@ func TestClientsAddExisting(t *testing.T) {
 	var c Client
 	clients := clientsContainer{}
 	clients.testing = true
-	clients.Init(nil, nil)
+	clients.Init(nil, nil, nil)
 
 	// some test variables
 	mac, _ := net.ParseMAC("aa:aa:aa:aa:aa:aa")

home/dns.go

@@ -56,6 +56,7 @@ func initDNSServer() error {
 		bindhost = "127.0.0.1"
 	}
 	filterConf.ResolverAddress = fmt.Sprintf("%s:%d", bindhost, config.DNS.Port)
+	filterConf.AutoHosts = &Context.autoHosts
 	filterConf.ConfigModified = onConfigModified
 	filterConf.HTTPRegister = httpRegister
 	Context.dnsFilter = dnsfilter.New(&filterConf, nil)

home/home.go

@@ -68,6 +68,7 @@ type homeContext struct {
 	filters    Filtering            // DNS filtering module
 	web        *Web                 // Web (HTTP, HTTPS) module
 	tls        *TLSMod              // TLS module
+	autoHosts  util.AutoHosts       // IP-hostname pairs taken from system configuration (e.g. /etc/hosts) files
 
 	// Runtime properties
 	// --
@@ -210,7 +211,8 @@ func run(args options) {
 	if Context.dhcpServer == nil {
 		os.Exit(1)
 	}
-	Context.clients.Init(config.Clients, Context.dhcpServer)
+	Context.autoHosts.Init("")
+	Context.clients.Init(config.Clients, Context.dhcpServer, &Context.autoHosts)
 	config.Clients = nil
 
 	if (runtime.GOOS == "linux" || runtime.GOOS == "darwin") &&
@@ -270,6 +272,7 @@ func run(args options) {
 			log.Fatalf("%s", err)
 		}
 		Context.tls.Start()
+		Context.autoHosts.Start()
 
 		go func() {
 			err := startDNSServer()
@@ -434,6 +437,8 @@ func cleanup() {
 		log.Error("Couldn't stop DHCP server: %s", err)
 	}
 
+	Context.autoHosts.Close()
+
 	if Context.tls != nil {
 		Context.tls.Close()
 		Context.tls = nil

home/service.go

@@ -196,7 +196,7 @@ func handleServiceInstallCommand(s service.Service) {
 		log.Fatal(err)
 	}
 
-	if isOpenWrt() {
+	if util.IsOpenWrt() {
 		// On OpenWrt it is important to run enable after the service installation
 		// Otherwise, the service won't start on the system startup
 		_, err := runInitdCommand("enable")
@@ -223,7 +223,7 @@ Click on the link below and follow the Installation Wizard steps to finish setup
 
 // handleServiceStatusCommand handles service "uninstall" command
 func handleServiceUninstallCommand(s service.Service) {
-	if isOpenWrt() {
+	if util.IsOpenWrt() {
 		// On OpenWrt it is important to run disable command first
 		// as it will remove the symlink
 		_, err := runInitdCommand("disable")
@@ -270,7 +270,7 @@ func configureService(c *service.Config) {
 	c.Option["SysvScript"] = sysvScript
 
 	// On OpenWrt we're using a different type of sysvScript
-	if isOpenWrt() {
+	if util.IsOpenWrt() {
 		c.Option["SysvScript"] = openWrtScript
 	}
 }
@@ -283,20 +283,6 @@ func runInitdCommand(action string) (int, error) {
 	return code, err
 }
 
-// isOpenWrt checks if OS is OpenWRT
-func isOpenWrt() bool {
-	if runtime.GOOS != "linux" {
-		return false
-	}
-
-	body, err := ioutil.ReadFile("/etc/os-release")
-	if err != nil {
-		return false
-	}
-
-	return strings.Contains(string(body), "OpenWrt")
-}
-
 // Basically the same template as the one defined in github.com/kardianos/service
 // but with two additional keys - StandardOutPath and StandardErrorPath
 var launchdConfig = `<?xml version='1.0' encoding='UTF-8'?>