Merge remote-tracking branch 'origin/master' into 1478-auto-records

AGHTechDoc.md

@@ -946,7 +946,7 @@ Response:
 	{
 		allowed_clients: ["127.0.0.1", ...]
 		disallowed_clients: ["127.0.0.1", ...]
-		blocked_hosts: ["host.com", ...]
+		blocked_hosts: ["host.com", ...] // host name or a wildcard
 	}
 
 
@@ -1287,12 +1287,22 @@ Request:
 	{
 		"enabled": true | false
 		"interval": 1 | 7 | 30 | 90
+		"anonymize_client_ip": true | false // anonymize clients' IP addresses
 	}
 
 Response:
 
 	200 OK
 
+`anonymize_client_ip`:
+1. New log entries written to a log file will contain modified client IP addresses.  Note that there's no way to obtain the full IP address later for these entries.
+2. `GET /control/querylog` response data will contain modified client IP addresses (masked /24 or /112).
+3. Searching by client IP won't work for the previously stored entries.
+
+How `anonymize_client_ip` affects Stats:
+1. After AGH restart, new stats entries will contain modified client IP addresses.
+2. Existing entries are not affected.
+
 
 ### API: Get querylog parameters
 
@@ -1307,6 +1317,7 @@ Response:
 	{
 		"enabled": true | false
 		"interval": 1 | 7 | 30 | 90
+		"anonymize_client_ip": true | false
 	}
 
 

client/src/__locales/en.json

@@ -199,6 +199,8 @@
     "query_log_disabled": "The query log is disabled and can be configured in the <0>settings</0>",
     "query_log_strict_search": "Use double quotes for strict search",
     "query_log_retention_confirm": "Are you sure you want to change query log retention? If you decrease the interval value, some data will be lost",
+    "anonymize_client_ip": "Anonymize client IP",
+    "anonymize_client_ip_desc": "Don't save the full IP address of the client in logs and statistics",
     "dns_config": "DNS server configuration",
     "blocking_mode": "Blocking mode",
     "default": "Default",
@@ -357,7 +359,7 @@
     "access_disallowed_title": "Disallowed clients",
     "access_disallowed_desc": "A list of CIDR or IP addresses. If configured, AdGuard Home will drop requests from these IP addresses.",
     "access_blocked_title": "Blocked domains",
-    "access_blocked_desc": "Don't confuse this with filters. AdGuard Home will drop DNS queries with these domains in query's question.",
+    "access_blocked_desc": "Don't confuse this with filters. AdGuard Home will drop DNS queries with these domains in query's question. Here you can specify the exact domain names, wildcards and urlfilter-rules, e.g. 'example.org', '*.example.org' or '||example.org^'.",
     "access_settings_saved": "Access settings successfully saved",
     "updates_checked": "Updates successfully checked",
     "updates_version_equal": "AdGuard Home is up-to-date",

client/src/components/Settings/Clients/Form.js

@@ -112,6 +112,7 @@ const renderMultiselect = (props) => {
             onChange={value => input.onChange(value)}
             onBlur={() => input.onBlur(input.value)}
             placeholder={placeholder}
+            blurInputOnSelect={false}
             isMulti
         />
     );

client/src/components/Settings/LogsConfig/Form.js

@@ -42,6 +42,16 @@ const Form = (props) => {
                     disabled={processing}
                 />
             </div>
+            <div className="form__group form__group--settings">
+                <Field
+                    name="anonymize_client_ip"
+                    type="checkbox"
+                    component={renderSelectField}
+                    placeholder={t('anonymize_client_ip')}
+                    subtitle={t('anonymize_client_ip_desc')}
+                    disabled={processing}
+                />
+            </div>
             <label className="form__label">
                 <Trans>query_log_retention</Trans>
             </label>

client/src/components/Settings/LogsConfig/index.js

@@ -30,7 +30,7 @@ class LogsConfig extends Component {
 
     render() {
         const {
-            t, enabled, interval, processing, processingClear,
+            t, enabled, interval, processing, processingClear, anonymize_client_ip,
         } = this.props;
 
         return (
@@ -44,6 +44,7 @@ class LogsConfig extends Component {
                         initialValues={{
                             enabled,
                             interval,
+                            anonymize_client_ip,
                         }}
                         onSubmit={this.handleFormSubmit}
                         processing={processing}
@@ -59,6 +60,7 @@ class LogsConfig extends Component {
 LogsConfig.propTypes = {
     interval: PropTypes.number.isRequired,
     enabled: PropTypes.bool.isRequired,
+    anonymize_client_ip: PropTypes.bool.isRequired,
     processing: PropTypes.bool.isRequired,
     processingClear: PropTypes.bool.isRequired,
     setLogsConfig: PropTypes.func.isRequired,

client/src/components/Settings/index.js

@@ -106,6 +106,7 @@ class Settings extends Component {
                                 <LogsConfig
                                     enabled={queryLogs.enabled}
                                     interval={queryLogs.interval}
+                                    anonymize_client_ip={queryLogs.anonymize_client_ip}
                                     processing={queryLogs.processingSetConfig}
                                     processingClear={queryLogs.processingClear}
                                     setLogsConfig={setLogsConfig}

client/src/reducers/queryLogs.js

@@ -134,6 +134,7 @@ const queryLogs = handleActions(
         oldest: '',
         filter: DEFAULT_LOGS_FILTER,
         isFiltered: false,
+        anonymize_client_ip: false,
     },
 );
 

dnsforward/access.go

@@ -2,11 +2,15 @@ package dnsforward
 
 import (
 	"encoding/json"
+	"fmt"
 	"net"
 	"net/http"
+	"strings"
 	"sync"
 
 	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/urlfilter"
+	"github.com/AdguardTeam/urlfilter/filterlist"
 )
 
 type accessCtx struct {
@@ -18,7 +22,7 @@ type accessCtx struct {
 	allowedClientsIPNet    []net.IPNet // CIDRs of whitelist clients
 	disallowedClientsIPNet []net.IPNet // CIDRs of clients that should be blocked
 
-	blockedHosts map[string]bool // hosts that should be blocked
+	blockedHostsEngine *urlfilter.DNSEngine // finds hosts that should be blocked
 }
 
 func (a *accessCtx) Init(allowedClients, disallowedClients, blockedHosts []string) error {
@@ -32,15 +36,26 @@ func (a *accessCtx) Init(allowedClients, disallowedClients, blockedHosts []strin
 		return err
 	}
 
-	convertArrayToMap(&a.blockedHosts, blockedHosts)
-	return nil
-}
+	buf := strings.Builder{}
+	for _, s := range blockedHosts {
+		buf.WriteString(s)
+		buf.WriteString("\n")
+	}
 
-func convertArrayToMap(dst *map[string]bool, src []string) {
-	*dst = make(map[string]bool)
-	for _, s := range src {
-		(*dst)[s] = true
+	listArray := []filterlist.RuleList{}
+	list := &filterlist.StringRuleList{
+		ID:             int(0),
+		RulesText:      buf.String(),
+		IgnoreCosmetic: true,
 	}
+	listArray = append(listArray, list)
+	rulesStorage, err := filterlist.NewRuleStorage(listArray)
+	if err != nil {
+		return fmt.Errorf("filterlist.NewRuleStorage(): %s", err)
+	}
+	a.blockedHostsEngine = urlfilter.NewDNSEngine(rulesStorage)
+
+	return nil
 }
 
 // Split array of IP or CIDR into 2 containers for fast search
@@ -107,7 +122,7 @@ func (a *accessCtx) IsBlockedIP(ip string) bool {
 // IsBlockedDomain - return TRUE if this domain should be blocked
 func (a *accessCtx) IsBlockedDomain(host string) bool {
 	a.lock.Lock()
-	_, ok := a.blockedHosts[host]
+	_, ok := a.blockedHostsEngine.Match(host, nil)
 	a.lock.Unlock()
 	return ok
 }

dnsforward/dnsforward_test.go

@@ -796,11 +796,26 @@ func TestIsBlockedIPDisallowed(t *testing.T) {
 
 func TestIsBlockedIPBlockedDomain(t *testing.T) {
 	a := &accessCtx{}
-	assert.True(t, a.Init(nil, nil, []string{"host1", "host2"}) == nil)
+	assert.True(t, a.Init(nil, nil, []string{"host1",
+		"host2",
+		"*.host.com",
+		"||host3.com^",
+	}) == nil)
 
+	// match by "host2.com"
 	assert.True(t, a.IsBlockedDomain("host1"))
 	assert.True(t, a.IsBlockedDomain("host2"))
 	assert.True(t, !a.IsBlockedDomain("host3"))
+
+	// match by wildcard "*.host.com"
+	assert.True(t, !a.IsBlockedDomain("host.com"))
+	assert.True(t, a.IsBlockedDomain("asdf.host.com"))
+	assert.True(t, a.IsBlockedDomain("qwer.asdf.host.com"))
+	assert.True(t, !a.IsBlockedDomain("asdf.zhost.com"))
+
+	// match by wildcard "||host3.com^"
+	assert.True(t, a.IsBlockedDomain("host3.com"))
+	assert.True(t, a.IsBlockedDomain("asdf.host3.com"))
 }
 
 func TestValidateUpstream(t *testing.T) {

home/config.go

@@ -77,9 +77,10 @@ type dnsConfig struct {
 	// time interval for statistics (in days)
 	StatsInterval uint32 `yaml:"statistics_interval"`
 
-	QueryLogEnabled  bool   `yaml:"querylog_enabled"`  // if true, query log is enabled
-	QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
-	QueryLogMemSize  uint32 `yaml:"querylog_memsize"`  // number of entries kept in memory before they are flushed to disk
+	QueryLogEnabled   bool   `yaml:"querylog_enabled"`     // if true, query log is enabled
+	QueryLogInterval  uint32 `yaml:"querylog_interval"`    // time interval for query log (in days)
+	QueryLogMemSize   uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
+	AnonymizeClientIP bool   `yaml:"anonymize_client_ip"`  // anonymize clients' IP addresses in logs and stats
 
 	dnsforward.FilteringConfig `yaml:",inline"`
 
@@ -242,6 +243,7 @@ func (c *configuration) write() error {
 		config.DNS.QueryLogEnabled = dc.Enabled
 		config.DNS.QueryLogInterval = dc.Interval
 		config.DNS.QueryLogMemSize = dc.MemSize
+		config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
 	}
 
 	if Context.dnsFilter != nil {

home/dns.go

@@ -29,22 +29,24 @@ func initDNSServer() error {
 	baseDir := Context.getDataDir()
 
 	statsConf := stats.Config{
-		Filename:       filepath.Join(baseDir, "stats.db"),
-		LimitDays:      config.DNS.StatsInterval,
-		ConfigModified: onConfigModified,
-		HTTPRegister:   httpRegister,
+		Filename:          filepath.Join(baseDir, "stats.db"),
+		LimitDays:         config.DNS.StatsInterval,
+		AnonymizeClientIP: config.DNS.AnonymizeClientIP,
+		ConfigModified:    onConfigModified,
+		HTTPRegister:      httpRegister,
 	}
 	Context.stats, err = stats.New(statsConf)
 	if err != nil {
 		return fmt.Errorf("Couldn't initialize statistics module")
 	}
 	conf := querylog.Config{
-		Enabled:        config.DNS.QueryLogEnabled,
-		BaseDir:        baseDir,
-		Interval:       config.DNS.QueryLogInterval,
-		MemSize:        config.DNS.QueryLogMemSize,
-		ConfigModified: onConfigModified,
-		HTTPRegister:   httpRegister,
+		Enabled:           config.DNS.QueryLogEnabled,
+		BaseDir:           baseDir,
+		Interval:          config.DNS.QueryLogInterval,
+		MemSize:           config.DNS.QueryLogMemSize,
+		AnonymizeClientIP: config.DNS.AnonymizeClientIP,
+		ConfigModified:    onConfigModified,
+		HTTPRegister:      httpRegister,
 	}
 	Context.queryLog = querylog.New(conf)
 

home/home_test.go

@@ -28,7 +28,7 @@ dns:
   statistics_interval: 90
   querylog_enabled: true
   querylog_interval: 90
-  querylog_memsize: 0
+  querylog_size_memory: 0
   protection_enabled: true
   blocking_mode: null_ip
   blocked_response_ttl: 0

openapi/openapi.yaml

@@ -1585,6 +1585,9 @@ definitions:
             interval:
                 type: "integer"
                 description: "Time period to keep data (1 | 7 | 30 | 90)"
+            anonymize_client_ip:
+                type: "boolean"
+                description: "Anonymize clients' IP addresses"
 
     TlsConfig:
         type: "object"

querylog/qlog.go

@@ -2,6 +2,7 @@ package querylog
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -65,6 +66,8 @@ func checkInterval(days uint32) bool {
 func (l *queryLog) WriteDiskConfig(dc *DiskConfig) {
 	dc.Enabled = l.conf.Enabled
 	dc.Interval = l.conf.Interval
+	dc.MemSize = l.conf.MemSize
+	dc.AnonymizeClientIP = l.conf.AnonymizeClientIP
 }
 
 // Clear memory buffer and remove log files
@@ -122,7 +125,7 @@ func (l *queryLog) Add(params AddParams) {
 
 	now := time.Now()
 	entry := logEntry{
-		IP:   params.ClientIP.String(),
+		IP:   l.getClientIP(params.ClientIP.String()),
 		Time: now,
 
 		Result:   *params.Result,
@@ -195,6 +198,10 @@ const (
 func (l *queryLog) getData(params getDataParams) map[string]interface{} {
 	now := time.Now()
 
+	if len(params.Client) != 0 && l.conf.AnonymizeClientIP {
+		params.Client = l.getClientIP(params.Client)
+	}
+
 	// add from file
 	fileEntries, oldest, total := l.searchFiles(params)
 
@@ -245,7 +252,7 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} {
 	// the elements order is already reversed (from newer to older)
 	for i := 0; i < len(entries); i++ {
 		entry := entries[i]
-		jsonEntry := logEntryToJSONEntry(entry)
+		jsonEntry := l.logEntryToJSONEntry(entry)
 		data = append(data, jsonEntry)
 	}
 
@@ -261,7 +268,26 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} {
 	return result
 }
 
-func logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
+// Get Client IP address
+func (l *queryLog) getClientIP(clientIP string) string {
+	if l.conf.AnonymizeClientIP {
+		ip := net.ParseIP(clientIP)
+		if ip != nil {
+			ip4 := ip.To4()
+			const AnonymizeClientIP4Mask = 24
+			const AnonymizeClientIP6Mask = 112
+			if ip4 != nil {
+				clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String()
+			} else {
+				clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String()
+			}
+		}
+	}
+
+	return clientIP
+}
+
+func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 	var msg *dns.Msg
 
 	if len(entry.Answer) > 0 {
@@ -276,7 +302,7 @@ func logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 		"reason":    entry.Result.Reason.String(),
 		"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
 		"time":      entry.Time.Format(time.RFC3339Nano),
-		"client":    entry.IP,
+		"client":    l.getClientIP(entry.IP),
 	}
 	jsonEntry["question"] = map[string]interface{}{
 		"host":  entry.QHost,