Pull request: 3772 cache upstreams

Merge in DNS/adguard-home from 3772-cache-upstreams to master

Updates AdguardTeam#3772.

Squashed commit of the following:

commit 809e874
Merge: 5774798 2d328ea
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Dec 7 17:15:59 2021 +0300

    Merge branch 'master' into 3772-cache-upstreams

commit 5774798
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Dec 7 16:48:38 2021 +0300

    client: fix formatting

commit 4b4bb66
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Dec 7 16:47:18 2021 +0300

    client: remove comment

commit a2a9f90
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Dec 7 16:45:31 2021 +0300

    client: handle cached upstream

commit a56804c
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 26 15:33:43 2021 +0300

    querylog: rm todo

commit 16c0a62
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 26 15:31:53 2021 +0300

    all: revise log of changes

commit 9f4b793
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Nov 25 17:23:42 2021 +0300

    WIP

commit 7fbc6d5
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Nov 25 17:03:42 2021 +0300

    all: imp upstream info in querylog

CHANGELOG.md

@@ -15,6 +15,8 @@ and this project adheres to
 
 ### Added
 
+- Upstream server information for responses from cache ([#3772]).  Note that old
+  log entries concerning cached responses won't include that information.
 - Finnish and Ukrainian translations.
 - Setting the timeout for IP address pinging in the "Fastest IP address" mode
   through the new `fastest_timeout` field in the configuration file ([#1992]).
@@ -47,6 +49,7 @@ and this project adheres to
 
 ### Changed
 
+- Responses from cache are now labeled ([#3772]).
 - Better error message for ED25519 private keys, which are not widely supported
   ([#3737]).
 - Cache now follows RFC more closely for negative answers ([#3707]).
@@ -223,6 +226,7 @@ In this release, the schema version has changed from 10 to 12.
 [#3655]: https://github.com/AdguardTeam/AdGuardHome/issues/3655
 [#3707]: https://github.com/AdguardTeam/AdGuardHome/issues/3707
 [#3744]: https://github.com/AdguardTeam/AdGuardHome/issues/3744
+[#3772]: https://github.com/AdguardTeam/AdGuardHome/issues/3772
 [#3815]: https://github.com/AdguardTeam/AdGuardHome/issues/3815
 [#3890]: https://github.com/AdguardTeam/AdGuardHome/issues/3890
 

client/src/__locales/en.json

@@ -47,7 +47,6 @@
     "form_error_server_name": "Invalid server name",
     "form_error_subnet": "Subnet \"{{cidr}}\" does not contain the IP address \"{{ip}}\"",
     "form_error_positive": "Must be greater than 0",
-    "form_error_negative": "Must be equal to 0 or greater",
     "out_of_range_error": "Must be out of range \"{{start}}\"-\"{{end}}\"",
     "lower_range_start_error": "Must be lower than range start",
     "greater_range_start_error": "Must be greater than range start",
@@ -627,5 +626,6 @@
     "experimental": "Experimental",
     "use_saved_key": "Use the previously saved key",
     "parental_control": "Parental control",
-    "safe_browsing": "Safe browsing"
+    "safe_browsing": "Safe browsing",
+    "served_from_cache": "{{value}} <i>(served from cache)</i>"
 }

client/src/components/Logs/Cells/ResponseCell.js

@@ -21,6 +21,7 @@ const ResponseCell = ({
     upstream,
     rules,
     service_name,
+    cached,
 }) => {
     const { t } = useTranslation();
     const filters = useSelector((state) => state.filtering.filters, shallowEqual);
@@ -36,6 +37,9 @@ const ResponseCell = ({
 
     const statusLabel = t(isBlockedByResponse ? 'blocked_by_cname_or_ip' : FILTERED_STATUS_TO_META_MAP[reason]?.LABEL || reason);
     const boldStatusLabel = <span className="font-weight-bold">{statusLabel}</span>;
+    const upstreamString = cached
+        ? t('served_from_cache', { value: upstream, i: <i /> })
+        : upstream;
 
     const renderResponses = (responseArr) => {
         if (!responseArr || responseArr.length === 0) {
@@ -53,7 +57,7 @@ const ResponseCell = ({
 
     const COMMON_CONTENT = {
         encryption_status: boldStatusLabel,
-        install_settings_dns: upstream,
+        install_settings_dns: upstreamString,
         elapsed: formattedElapsedMs,
         response_code: status,
         ...(service_name
@@ -90,8 +94,9 @@ const ResponseCell = ({
 
     const detailedInfo = getDetailedInfo(reason);
 
-    return <div className="logs__cell logs__cell--response" role="gridcell">
-        <IconTooltip
+    return (
+        <div className="logs__cell logs__cell--response" role="gridcell">
+            <IconTooltip
                 className={classNames('icons mr-4 icon--24 icon--lightgray', { 'my-3': isDetailed })}
                 columnClass='grid grid--limited'
                 tooltipClass='px-5 pb-5 pt-4 mw-75 custom-tooltip__response-details'
@@ -100,14 +105,15 @@ const ResponseCell = ({
                 title='response_details'
                 content={content}
                 placement='bottom'
-        />
-        <div className="text-truncate">
-            <div className="text-truncate" title={statusLabel}>{statusLabel}</div>
-            {isDetailed && <div
-                    className="detailed-info d-none d-sm-block pt-1 text-truncate"
-                    title={detailedInfo}>{detailedInfo}</div>}
+            />
+            <div className="text-truncate">
+                <div className="text-truncate" title={statusLabel}>{statusLabel}</div>
+                {isDetailed && <div
+                        className="detailed-info d-none d-sm-block pt-1 text-truncate"
+                        title={detailedInfo}>{detailedInfo}</div>}
+            </div>
         </div>
-    </div>;
+    );
 };
 
 ResponseCell.propTypes = {
@@ -117,6 +123,7 @@ ResponseCell.propTypes = {
     response: propTypes.array.isRequired,
     status: propTypes.string.isRequired,
     upstream: propTypes.string.isRequired,
+    cached: propTypes.bool.isRequired,
     rules: propTypes.arrayOf(propTypes.shape({
         text: propTypes.string.isRequired,
         filter_list_id: propTypes.number.isRequired,

client/src/components/Logs/Cells/index.js

@@ -76,6 +76,7 @@ const Row = memo(({
             originalResponse,
             status,
             service_name,
+            cached,
         } = rowProps;
 
         const hasTracker = !!tracker;
@@ -116,6 +117,9 @@ const Row = memo(({
 
         const blockingForClientKey = isFiltered ? 'unblock_for_this_client_only' : 'block_for_this_client_only';
         const clientNameBlockingFor = getBlockingClientName(clients, client);
+        const upstreamString = cached
+            ? t('served_from_cache', { value: upstream, i: <i /> })
+            : upstream;
 
         const onBlockingForClientClick = () => {
             dispatch(toggleBlockingForClient(buttonType, domain, clientNameBlockingFor));
@@ -175,7 +179,7 @@ const Row = memo(({
                             className="link--green">{sourceData.name}
                     </a>,
             response_details: 'title',
-            install_settings_dns: upstream,
+            install_settings_dns: upstreamString,
             elapsed: formattedElapsedMs,
             ...(rules.length > 0
                     && { rule_label: getRulesToFilterList(rules, filters, whitelistFilters) }
@@ -230,6 +234,7 @@ Row.propTypes = {
         time: propTypes.string.isRequired,
         tracker: propTypes.object,
         upstream: propTypes.string.isRequired,
+        cached: propTypes.bool.isRequired,
         type: propTypes.string.isRequired,
         client_proto: propTypes.string.isRequired,
         client_id: propTypes.string,

client/src/helpers/helpers.js

@@ -75,6 +75,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
         service_name,
         original_answer,
         upstream,
+        cached,
     } = log;
 
     const { name: domain, unicode_name: unicodeName, type } = question;
@@ -116,6 +117,7 @@ export const normalizeLogs = (logs) => logs.map((log) => {
         answer_dnssec,
         elapsedMs,
         upstream,
+        cached,
     };
 });
 

client/src/helpers/validators.js

@@ -230,17 +230,6 @@ export const validateMac = (value) => {
     return undefined;
 };
 
-/**
- * @param value {number}
- * @returns {boolean|*}
- */
-export const validateBiggerOrEqualZeroValue = (value) => {
-    if (value < 0) {
-        return 'form_error_negative';
-    }
-    return false;
-};
-
 /**
  * @param value {number}
  * @returns {undefined|string}

go.mod

@@ -3,8 +3,8 @@ module github.com/AdguardTeam/AdGuardHome
 go 1.16
 
 require (
-	github.com/AdguardTeam/dnsproxy v0.39.9
-	github.com/AdguardTeam/golibs v0.10.2
+	github.com/AdguardTeam/dnsproxy v0.39.12
+	github.com/AdguardTeam/golibs v0.10.3
 	github.com/AdguardTeam/urlfilter v0.15.0
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.2.2

go.sum

@@ -9,13 +9,13 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AdguardTeam/dhcp v0.0.0-20210519141215-51808c73c0bf h1:gc042VRSIRSUzZ+Px6xQCRWNJZTaPkomisDfUZmoFNk=
 github.com/AdguardTeam/dhcp v0.0.0-20210519141215-51808c73c0bf/go.mod h1:TKl4jN3Voofo4UJIicyNhWGp/nlQqQkFxmwIFTvBkKI=
-github.com/AdguardTeam/dnsproxy v0.39.9 h1:lH4lKA7KHKFJZgzlij1YAVX6v7eIQpUFpYh9qV+WfGw=
-github.com/AdguardTeam/dnsproxy v0.39.9/go.mod h1:eDpJKAdkHORRwAedjuERv+7SWlcz4cn+5uwrbUAWHRY=
+github.com/AdguardTeam/dnsproxy v0.39.12 h1:BxAfdQLGnu0rqhD23K5nNw09sKQeqpT2AkFEE78WOqU=
+github.com/AdguardTeam/dnsproxy v0.39.12/go.mod h1:g7zjF1TWpKNeDVh6h3YrjQN8565zsWRd7zo++C/935c=
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.4.2/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.9.2/go.mod h1:fCAMwPBJ8S7YMYbTWvYS+eeTLblP5E04IDtNAo7y7IY=
-github.com/AdguardTeam/golibs v0.10.2 h1:TAwnS4Y49sSUa4UX1yz/MWNGbIlXHqafrWr9MxdIh9A=
-github.com/AdguardTeam/golibs v0.10.2/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
+github.com/AdguardTeam/golibs v0.10.3 h1:FBgk17zf35ESVWQKIqEUiqqB2bDaCBC8X5vMU760yB4=
+github.com/AdguardTeam/golibs v0.10.3/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
 github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
 github.com/AdguardTeam/urlfilter v0.15.0 h1:K3WWZE0K5nPTHe2l+TRXDFpYWJJnvkHdlWidt6NQUTk=
 github.com/AdguardTeam/urlfilter v0.15.0/go.mod h1:EwXwrYhowP7bedqmOrmKKmQtpBYFyDNEBFQ+lxdUgQU=
@@ -64,7 +64,6 @@ github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020 h1:mdi6AbCEoKCA1xKCmp7UtRB5fvGFlP92PvlhxgdvXEw=
 github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020/go.mod h1:KmHOjTUmJh/l04ukqPoBWPEZr9jwN05h5NXQl5C+DyY=
-github.com/go-test/deep v1.0.5 h1:AKODKU3pDH1RzZzm6YZu77YWtEAq6uh1rLIAQlay2qc=
 github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

internal/dnsforward/stats.go

@@ -27,11 +27,12 @@ func (s *Server) processQueryLogsAndStats(ctx *dnsContext) (rc resultCode) {
 		shouldLog = false
 	}
 
+	ip, _ := netutil.IPAndPortFromAddr(pctx.Addr)
+	ip = netutil.CloneIP(ip)
+
 	s.serverLock.RLock()
 	defer s.serverLock.RUnlock()
 
-	ip, _ := netutil.IPAndPortFromAddr(pctx.Addr)
-	ip = netutil.CloneIP(ip)
 	s.anonymizer.Load()(ip)
 
 	log.Debug("client ip: %s", ip)
@@ -60,12 +61,14 @@ func (s *Server) processQueryLogsAndStats(ctx *dnsContext) (rc resultCode) {
 		case proxy.ProtoDNSCrypt:
 			p.ClientProto = querylog.ClientProtoDNSCrypt
 		default:
-			// Consider this a plain DNS-over-UDP or DNS-over-TCP
-			// request.
+			// Consider this a plain DNS-over-UDP or DNS-over-TCP request.
 		}
 
 		if pctx.Upstream != nil {
 			p.Upstream = pctx.Upstream.Address()
+		} else if cachedUps := pctx.CachedUpstreamAddr; cachedUps != "" {
+			p.Upstream = pctx.CachedUpstreamAddr
+			p.Cached = true
 		}
 
 		s.queryLog.Add(p)

internal/querylog/decode.go

@@ -44,8 +44,10 @@ var logEntryHandlers = map[string]logEntryHandler{
 		if !ok {
 			return nil
 		}
+
 		var err error
 		ent.Time, err = time.Parse(time.RFC3339, v)
+
 		return err
 	},
 	"QH": func(t json.Token, ent *logEntry) error {
@@ -69,54 +71,77 @@ var logEntryHandlers = map[string]logEntryHandler{
 		if !ok {
 			return nil
 		}
+
 		ent.QClass = v
+
 		return nil
 	},
 	"CP": func(t json.Token, ent *logEntry) error {
 		v, ok := t.(string)
 		if !ok {
 			return nil
 		}
+
 		var err error
 		ent.ClientProto, err = NewClientProto(v)
+
 		return err
 	},
 	"Answer": func(t json.Token, ent *logEntry) error {
 		v, ok := t.(string)
 		if !ok {
 			return nil
 		}
+
 		var err error
 		ent.Answer, err = base64.StdEncoding.DecodeString(v)
+
 		return err
 	},
 	"OrigAnswer": func(t json.Token, ent *logEntry) error {
 		v, ok := t.(string)
 		if !ok {
 			return nil
 		}
+
 		var err error
 		ent.OrigAnswer, err = base64.StdEncoding.DecodeString(v)
+
 		return err
 	},
+	"Cached": func(t json.Token, ent *logEntry) error {
+		v, ok := t.(bool)
+		if !ok {
+			return nil
+		}
+
+		ent.Cached = v
+
+		return nil
+	},
 	"Upstream": func(t json.Token, ent *logEntry) error {
 		v, ok := t.(string)
 		if !ok {
 			return nil
 		}
+
 		ent.Upstream = v
+
 		return nil
 	},
 	"Elapsed": func(t json.Token, ent *logEntry) error {
 		v, ok := t.(json.Number)
 		if !ok {
 			return nil
 		}
+
 		i, err := v.Int64()
 		if err != nil {
 			return err
 		}
+
 		ent.Elapsed = time.Duration(i)
+
 		return nil
 	},
 }

internal/querylog/decode_test.go

@@ -33,6 +33,7 @@ func TestDecodeLogEntry(t *testing.T) {
 			`"QC":"IN",` +
 			`"CP":"",` +
 			`"Answer":"` + ansStr + `",` +
+			`"Cached":true,` +
 			`"Result":{` +
 			`"IsFiltered":true,` +
 			`"Reason":3,` +
@@ -42,6 +43,7 @@ func TestDecodeLogEntry(t *testing.T) {
 			`"CanonName":"example.com",` +
 			`"ServiceName":"example.org",` +
 			`"DNSRewriteResult":{"RCode":0,"Response":{"1":["127.0.0.2"]}}},` +
+			`"Upstream":"https://some.upstream",` +
 			`"Elapsed":837429}`
 
 		ans, err := base64.StdEncoding.DecodeString(ansStr)
@@ -56,6 +58,7 @@ func TestDecodeLogEntry(t *testing.T) {
 			ClientID:    "cli42",
 			ClientProto: "",
 			Answer:      ans,
+			Cached:      true,
 			Result: filtering.Result{
 				IsFiltered: true,
 				Reason:     filtering.FilteredBlockList,
@@ -78,7 +81,8 @@ func TestDecodeLogEntry(t *testing.T) {
 					},
 				},
 			},
-			Elapsed: 837429,
+			Upstream: "https://some.upstream",
+			Elapsed:  837429,
 		}
 
 		got := &logEntry{}

internal/querylog/json.go

@@ -74,6 +74,7 @@ func (l *queryLog) entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (json
 		"time":         entry.Time.Format(time.RFC3339Nano),
 		"client":       eip,
 		"client_proto": entry.ClientProto,
+		"cached":       entry.Cached,
 		"upstream":     entry.Upstream,
 		"question":     question,
 	}