Skip to content

Commit

Permalink
Pull request: all: add the $dnsrewrite modifier
Browse files Browse the repository at this point in the history 
Merge in DNS/urlfilter from 2102-dnsrewrite to master

Updates AdguardTeam/AdGuardHome#2102.

Squashed commit of the following:

commit 44ef59e
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Dec 18 16:27:01 2020 +0300

    all: improve tests

commit f863a56
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Dec 18 15:14:06 2020 +0300

    all: improve matching api

commit 6d5a48d
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Dec 14 16:41:19 2020 +0300

    rules: improve code

commit d0ff0b9
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Dec 11 20:40:42 2020 +0300

    all: improve performance, add more tests

commit 1f16df5
Merge: e4be9a4 ad2a7d2
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Dec 11 19:53:41 2020 +0300

    Merge branch 'master' into 2102-dnsrewrite

commit e4be9a4
Merge: f80f32b e5d7af3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Dec 10 20:20:26 2020 +0300

    Merge branch 'master' into 2102-dnsrewrite

commit f80f32b
Merge: 264c0ca e948b30
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Dec 10 16:39:38 2020 +0300

    Merge branch 'master' into 2102-dnsrewrite

commit 264c0ca
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Dec 10 15:57:37 2020 +0300

    all: improve matching

commit d33afd8
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Dec 8 18:50:13 2020 +0300

    all: improve api

commit dfe7e0e
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Dec 8 13:53:06 2020 +0300

    all: improve naming, tweak algorithms

commit 964d234
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Dec 4 16:56:29 2020 +0300

    all: add the $dnsrewrite modifier
  • Loading branch information
@ainar-g
ainar-g committed Dec 18, 2020
1 parent ad2a7d2 commit d0665c1
Show file tree
Hide file tree
Showing 20 changed files with 1,052 additions and 139 deletions.
2 changes: 1 addition & 1 deletion cosmetic_engine.go → cosmeticengine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ type CosmeticResult struct {
// includeJS defines if we should inject JS into the page (see $jsinject)
// includeGenericCSS defines if we should inject generic CSS and element hiding rules (see $generichide)
// TODO: Additionally, we should provide a method that writes result to an io.Writer
func (e *CosmeticEngine) Match(hostname string, includeCSS bool, includeJS bool, includeGenericCSS bool) CosmeticResult {
func (e *CosmeticEngine) Match(hostname string, includeCSS, includeJS, includeGenericCSS bool) CosmeticResult {
r := CosmeticResult{
ElementHiding: StylesResult{},
CSS: StylesResult{},
Expand Down
0 cosmetic_engine_test.go → cosmeticengine_test.go
View file
File renamed without changes.
46 changes: 28 additions & 18 deletions dns_engine.go → dnsengine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ type DNSResult struct {
NetworkRule *rules.NetworkRule // a network rule or nil
HostRulesV4 []*rules.HostRule // host rules for IPv4 or nil
HostRulesV6 []*rules.HostRule // host rules for IPv6 or nil

// networkRules are all matched network rules.
networkRules []*rules.NetworkRule
}

// DNSRequest represents a DNS query with associated metadata.
Expand All @@ -35,7 +38,7 @@ type DNSRequest struct {
// DNSType is the type of the resource record (RR) of a DNS request, for
// example "A" or "AAAA". See package github.com/miekg/dns for all
// acceptable constants.
DNSType uint16
DNSType rules.RRType
}

// NewDNSEngine parses the specified filter lists and returns a DNSEngine built from them.
Expand Down Expand Up @@ -63,9 +66,9 @@ func NewDNSEngine(s *filterlist.RuleStorage) *DNSEngine {

networkEngine := &NetworkEngine{
ruleStorage: s,
domainsLookupTable: make(map[uint32][]int64, 0),
domainsLookupTable: make(map[uint32][]int64),
shortcutsLookupTable: make(map[uint32][]int64, networkRulesCount),
shortcutsHistogram: make(map[uint32]int, 0),
shortcutsHistogram: make(map[uint32]int),
}

// Go through all rules in the storage and add them to the lookup tables
Expand Down Expand Up @@ -98,16 +101,17 @@ func (d *DNSEngine) Match(hostname string) (DNSResult, bool) {
return d.MatchRequest(DNSRequest{Hostname: hostname, ClientIP: "0.0.0.0"})
}

// MatchRequest - matches the specified DNS request
// MatchRequest matches the specified DNS request. The return parameter
// matched is true if the result has a basic network rule or some host
// rules.
//
// It returns true and the list of rules found or false and nil.
// The list of rules can be found when there're multiple host rules matching the same domain.
// For instance:
// 192.168.0.1 example.local
// 2000::1 example.local
func (d *DNSEngine) MatchRequest(dReq DNSRequest) (DNSResult, bool) {
// For compatibility reasons, it is also false when there are DNS
// rewrite and other kinds of special network rules, so users who need
// those will need to ignore the matched return parameter and instead
// inspect the results of the corresponding DNSResult getters.
func (d *DNSEngine) MatchRequest(dReq DNSRequest) (res DNSResult, matched bool) {
if dReq.Hostname == "" {
return DNSResult{}, false
return res, false
}

r := rules.NewRequestForHostname(dReq.Hostname)
Expand All @@ -116,28 +120,34 @@ func (d *DNSEngine) MatchRequest(dReq DNSRequest) (DNSResult, bool) {
r.ClientName = dReq.ClientName
r.DNSType = dReq.DNSType

networkRule, ok := d.networkEngine.Match(r)
if ok {
// Network rules always have higher priority
return DNSResult{NetworkRule: networkRule}, true
res.networkRules = d.networkEngine.MatchAll(r)

result := rules.NewMatchingResult(res.networkRules, nil)
resultRule := result.GetBasicResult()
if resultRule != nil {
// Network rules always have higher priority.
res.NetworkRule = resultRule
return res, true
}

rr, ok := d.matchLookupTable(dReq.Hostname)
if !ok {
return DNSResult{}, false
return res, false
}
res := DNSResult{}

for _, rule := range rr {
hostRule, ok := rule.(*rules.HostRule)
if !ok {
continue
}

if hostRule.IP.To4() != nil {
res.HostRulesV4 = append(res.HostRulesV4, hostRule)
} else {
res.HostRulesV6 = append(res.HostRulesV6, hostRule)
}
}

return res, true
}

Expand All @@ -164,7 +174,7 @@ func (d *DNSEngine) matchLookupTable(hostname string) ([]rules.Rule, bool) {
func (d *DNSEngine) addRule(hostRule *rules.HostRule, storageIdx int64) {
for _, hostname := range hostRule.Hostnames {
hash := fastHash(hostname)
rulesIndexes, _ := d.lookupTable[hash]
rulesIndexes := d.lookupTable[hash]
d.lookupTable[hash] = append(rulesIndexes, storageIdx)
}

Expand Down
114 changes: 57 additions & 57 deletions dns_engine_test.go → dnsengine_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func TestBenchDNSEngine(t *testing.T) {
if err != nil {
t.Fatalf("cannot create rule storage: %s", err)
}
defer ruleStorage.Close()
defer func() { assert.Nil(t, ruleStorage.Close()) }()

testRequests := loadRequests(t)
assert.True(t, len(testRequests) > 0)
Expand Down Expand Up @@ -197,7 +197,7 @@ func TestRegexp(t *testing.T) {
dnsEngine = NewDNSEngine(ruleStorage)

res, ok = dnsEngine.Match("stats.test.com")
assert.True(t, res.NetworkRule.Text() == text && res.NetworkRule.Whitelist)
assert.True(t, ok && res.NetworkRule.Text() == text && res.NetworkRule.Whitelist)
}

func TestClientTags(t *testing.T) {
Expand All @@ -219,63 +219,63 @@ func TestClientTags(t *testing.T) {
assert.NotNil(t, dnsEngine)

// global rule
rules, ok := dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", SortedClientTags: []string{"phone"}})
res, ok := dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", SortedClientTags: []string{"phone"}})
assert.True(t, ok)
assert.NotNil(t, rules.NetworkRule)
assert.Equal(t, "||host1^", rules.NetworkRule.Text())
assert.NotNil(t, res.NetworkRule)
assert.Equal(t, "||host1^", res.NetworkRule.Text())

// $ctag rule overrides global rule
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", SortedClientTags: []string{"pc"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", SortedClientTags: []string{"pc"}})
assert.True(t, ok)
assert.NotNil(t, rules.NetworkRule)
assert.Equal(t, "||host1^$ctag=pc|printer", rules.NetworkRule.Text())
assert.NotNil(t, res.NetworkRule)
assert.Equal(t, "||host1^$ctag=pc|printer", res.NetworkRule.Text())

// 1 tag matches
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", SortedClientTags: []string{"phone", "printer"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", SortedClientTags: []string{"phone", "printer"}})
assert.True(t, ok)
assert.NotNil(t, rules.NetworkRule)
assert.Equal(t, "||host2^$ctag=pc|printer", rules.NetworkRule.Text())
assert.NotNil(t, res.NetworkRule)
assert.Equal(t, "||host2^$ctag=pc|printer", res.NetworkRule.Text())

// tags don't match
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", SortedClientTags: []string{"phone"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", SortedClientTags: []string{"phone"}})
assert.False(t, ok)

// tags don't match
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", SortedClientTags: []string{}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", SortedClientTags: []string{}})
assert.False(t, ok)

// 1 tag matches (exclusion)
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", SortedClientTags: []string{"phone", "printer"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", SortedClientTags: []string{"phone", "printer"}})
assert.True(t, ok)
assert.NotNil(t, rules.NetworkRule)
assert.Equal(t, "||host3^$ctag=~pc|~router", rules.NetworkRule.Text())
assert.NotNil(t, res.NetworkRule)
assert.Equal(t, "||host3^$ctag=~pc|~router", res.NetworkRule.Text())

// 1 tag matches (exclusion)
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", SortedClientTags: []string{"phone", "router"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", SortedClientTags: []string{"phone", "router"}})
assert.True(t, ok)
assert.NotNil(t, rules.NetworkRule)
assert.Equal(t, "||host4^$ctag=~pc|router", rules.NetworkRule.Text())
assert.NotNil(t, res.NetworkRule)
assert.Equal(t, "||host4^$ctag=~pc|router", res.NetworkRule.Text())

// tags don't match (exclusion)
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", SortedClientTags: []string{"pc"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", SortedClientTags: []string{"pc"}})
assert.False(t, ok)

// tags don't match (exclusion)
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", SortedClientTags: []string{"pc", "router"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", SortedClientTags: []string{"pc", "router"}})
assert.False(t, ok)

// tags match but it's a $badfilter
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host5", SortedClientTags: []string{"pc"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host5", SortedClientTags: []string{"pc"}})
assert.False(t, ok)

// tags match and $badfilter rule disables global rule
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host6", SortedClientTags: []string{"pc"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host6", SortedClientTags: []string{"pc"}})
assert.True(t, ok)
assert.NotNil(t, rules.NetworkRule)
assert.Equal(t, "||host6^$ctag=pc|printer", rules.NetworkRule.Text())
assert.NotNil(t, res.NetworkRule)
assert.Equal(t, "||host6^$ctag=pc|printer", res.NetworkRule.Text())

// tags match (exclusion) but it's a $badfilter
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host7", SortedClientTags: []string{"phone"}})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host7", SortedClientTags: []string{"phone"}})
assert.False(t, ok)
}

Expand Down Expand Up @@ -304,75 +304,75 @@ func TestClient(t *testing.T) {
assert.NotNil(t, dnsEngine)

// match client IPv4
rules, ok := dnsEngine.MatchRequest(DNSRequest{Hostname: "host0", ClientIP: "127.0.0.1"})
assertMatchRuleText(t, rulesText[0], rules, ok)
res, ok := dnsEngine.MatchRequest(DNSRequest{Hostname: "host0", ClientIP: "127.0.0.1"})
assertMatchRuleText(t, rulesText[0], res, ok)

// not match client IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host0", ClientIP: "127.0.0.2"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host0", ClientIP: "127.0.0.2"})
assert.False(t, ok)

// restricted client IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", ClientIP: "127.0.0.1"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", ClientIP: "127.0.0.1"})
assert.False(t, ok)

// non-restricted client IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", ClientIP: "127.0.0.2"})
assertMatchRuleText(t, rulesText[1], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host1", ClientIP: "127.0.0.2"})
assertMatchRuleText(t, rulesText[1], res, ok)

// match client IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", ClientIP: "2001::c0:ffee"})
assertMatchRuleText(t, rulesText[2], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", ClientIP: "2001::c0:ffee"})
assertMatchRuleText(t, rulesText[2], res, ok)

// not match client IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", ClientIP: "2001::c0:ffef"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host2", ClientIP: "2001::c0:ffef"})
assert.False(t, ok)

// restricted client IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", ClientIP: "2001::c0:ffee"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", ClientIP: "2001::c0:ffee"})
assert.False(t, ok)

// non-restricted client IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", ClientIP: "2001::c0:ffef"})
assertMatchRuleText(t, rulesText[3], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host3", ClientIP: "2001::c0:ffef"})
assertMatchRuleText(t, rulesText[3], res, ok)

// match network IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", ClientIP: "127.0.0.254"})
assertMatchRuleText(t, rulesText[4], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", ClientIP: "127.0.0.254"})
assertMatchRuleText(t, rulesText[4], res, ok)

// not match network IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", ClientIP: "127.0.1.1"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host4", ClientIP: "127.0.1.1"})
assert.False(t, ok)

// restricted network IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host5", ClientIP: "127.0.0.254"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host5", ClientIP: "127.0.0.254"})
assert.False(t, ok)

// non-restricted network IPv4
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host5", ClientIP: "127.0.1.1"})
assertMatchRuleText(t, rulesText[5], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host5", ClientIP: "127.0.1.1"})
assertMatchRuleText(t, rulesText[5], res, ok)

// match network IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host6", ClientIP: "2001::c0:ff07"})
assertMatchRuleText(t, rulesText[6], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host6", ClientIP: "2001::c0:ff07"})
assertMatchRuleText(t, rulesText[6], res, ok)

// not match network IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host6", ClientIP: "2001::c0:feee"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host6", ClientIP: "2001::c0:feee"})
assert.False(t, ok)

// restricted network IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host7", ClientIP: "2001::c0:ff07"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host7", ClientIP: "2001::c0:ff07"})
assert.False(t, ok)

// non-restricted network IPv6
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host7", ClientIP: "2001::c0:feee"})
assertMatchRuleText(t, rulesText[7], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host7", ClientIP: "2001::c0:feee"})
assertMatchRuleText(t, rulesText[7], res, ok)

// match client name
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host8", ClientName: "Frank's laptop"})
assertMatchRuleText(t, rulesText[8], rules, ok)
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host8", ClientName: "Frank's laptop"})
assertMatchRuleText(t, rulesText[8], res, ok)

// not match client name
rules, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host8", ClientName: "Franks laptop"})
res, ok = dnsEngine.MatchRequest(DNSRequest{Hostname: "host8", ClientName: "Franks laptop"})
assert.False(t, ok)
}

Expand Down Expand Up @@ -519,17 +519,17 @@ func TestDNSEngine_MatchRequest_dnsType(t *testing.T) {
ClientIP: "127.0.0.1",
}

rules, ok := dnsEngine.MatchRequest(r)
res, ok := dnsEngine.MatchRequest(r)
assert.True(t, ok)
assert.Contains(t, rules.NetworkRule.Text(), "dnstype=")
assert.Contains(t, res.NetworkRule.Text(), "dnstype=")

r = DNSRequest{
Hostname: "priority",
DNSType: dns.TypeA,
ClientIP: "127.0.0.1",
}
rules, ok = dnsEngine.MatchRequest(r)
res, ok = dnsEngine.MatchRequest(r)
assert.True(t, ok)
assert.NotContains(t, rules.NetworkRule.Text(), "dnstype=")
assert.NotContains(t, res.NetworkRule.Text(), "dnstype=")
})
}
Loading

0 comments on commit d0665c1

Please sign in to comment.