From 72a8839d6624977b49bd47adadfa72e7432a482f Mon Sep 17 00:00:00 2001 From: Eugene Burkov Date: Tue, 7 Dec 2021 17:43:51 +0300 Subject: [PATCH] Pull request: 3772 cache upstreams Merge in DNS/adguard-home from 3772-cache-upstreams to master Updates #3772. Squashed commit of the following: commit 809e874fe8847fdbd7f8a7221f18393509f41339 Merge: 5774798c 2d328ea8 Author: Eugene Burkov Date: Tue Dec 7 17:15:59 2021 +0300 Merge branch 'master' into 3772-cache-upstreams commit 5774798ce1897bf292be49410794ce06cd084f93 Author: Ildar Kamalov Date: Tue Dec 7 16:48:38 2021 +0300 client: fix formatting commit 4b4bb668c762c9b4d47cf40a007f34fe47a6e260 Author: Ildar Kamalov Date: Tue Dec 7 16:47:18 2021 +0300 client: remove comment commit a2a9f904a6ab52194bb811a2a83967660a25f2cc Author: Ildar Kamalov Date: Tue Dec 7 16:45:31 2021 +0300 client: handle cached upstream commit a56804c4910da838cb10c8ef0af2a525e9722b1c Author: Eugene Burkov Date: Fri Nov 26 15:33:43 2021 +0300 querylog: rm todo commit 16c0a62ad2dfc7fcb5a6adbc868b296b7840fd27 Author: Eugene Burkov Date: Fri Nov 26 15:31:53 2021 +0300 all: revise log of changes commit 9f4b793205f17b6e85d74e83126093ed09eb24e3 Author: Eugene Burkov Date: Thu Nov 25 17:23:42 2021 +0300 WIP commit 7fbc6d596295570d2c61c3e507160f65c9adccb8 Author: Eugene Burkov Date: Thu Nov 25 17:03:42 2021 +0300 all: imp upstream info in querylog --- CHANGELOG.md | 4 +++ client/src/__locales/en.json | 4 +-- .../src/components/Logs/Cells/ResponseCell.js | 27 ++++++++++++------- client/src/components/Logs/Cells/index.js | 7 ++++- client/src/helpers/helpers.js | 2 ++ client/src/helpers/validators.js | 11 -------- go.mod | 4 +-- go.sum | 9 +++---- internal/dnsforward/stats.go | 11 +++++--- internal/querylog/decode.go | 25 +++++++++++++++++ internal/querylog/decode_test.go | 6 ++++- internal/querylog/json.go | 1 + internal/querylog/qlog.go | 4 ++- internal/querylog/querylog.go | 26 +++++++++++------- openapi/CHANGELOG.md | 5 ++++ openapi/openapi.yaml | 4 +++ 16 files changed, 104 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb638818cc..044b96885bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to ### Added +- Upstream server information for responses from cache ([#3772]). Note that old + log entries concerning cached responses won't include that information. - Finnish and Ukrainian translations. - Setting the timeout for IP address pinging in the "Fastest IP address" mode through the new `fastest_timeout` field in the configuration file ([#1992]). @@ -47,6 +49,7 @@ and this project adheres to ### Changed +- Responses from cache are now labeled ([#3772]). - Better error message for ED25519 private keys, which are not widely supported ([#3737]). - Cache now follows RFC more closely for negative answers ([#3707]). @@ -223,6 +226,7 @@ In this release, the schema version has changed from 10 to 12. [#3655]: https://github.com/AdguardTeam/AdGuardHome/issues/3655 [#3707]: https://github.com/AdguardTeam/AdGuardHome/issues/3707 [#3744]: https://github.com/AdguardTeam/AdGuardHome/issues/3744 +[#3772]: https://github.com/AdguardTeam/AdGuardHome/issues/3772 [#3815]: https://github.com/AdguardTeam/AdGuardHome/issues/3815 [#3890]: https://github.com/AdguardTeam/AdGuardHome/issues/3890 diff --git a/client/src/__locales/en.json b/client/src/__locales/en.json index 9caf8845497..edf63255074 100644 --- a/client/src/__locales/en.json +++ b/client/src/__locales/en.json @@ -47,7 +47,6 @@ "form_error_server_name": "Invalid server name", "form_error_subnet": "Subnet \"{{cidr}}\" does not contain the IP address \"{{ip}}\"", "form_error_positive": "Must be greater than 0", - "form_error_negative": "Must be equal to 0 or greater", "out_of_range_error": "Must be out of range \"{{start}}\"-\"{{end}}\"", "lower_range_start_error": "Must be lower than range start", "greater_range_start_error": "Must be greater than range start", @@ -627,5 +626,6 @@ "experimental": "Experimental", "use_saved_key": "Use the previously saved key", "parental_control": "Parental control", - "safe_browsing": "Safe browsing" + "safe_browsing": "Safe browsing", + "served_from_cache": "{{value}} (served from cache)" } diff --git a/client/src/components/Logs/Cells/ResponseCell.js b/client/src/components/Logs/Cells/ResponseCell.js index 3a3aeb60e53..772b89e5d7b 100644 --- a/client/src/components/Logs/Cells/ResponseCell.js +++ b/client/src/components/Logs/Cells/ResponseCell.js @@ -21,6 +21,7 @@ const ResponseCell = ({ upstream, rules, service_name, + cached, }) => { const { t } = useTranslation(); const filters = useSelector((state) => state.filtering.filters, shallowEqual); @@ -36,6 +37,9 @@ const ResponseCell = ({ const statusLabel = t(isBlockedByResponse ? 'blocked_by_cname_or_ip' : FILTERED_STATUS_TO_META_MAP[reason]?.LABEL || reason); const boldStatusLabel = {statusLabel}; + const upstreamString = cached + ? t('served_from_cache', { value: upstream, i: }) + : upstream; const renderResponses = (responseArr) => { if (!responseArr || responseArr.length === 0) { @@ -53,7 +57,7 @@ const ResponseCell = ({ const COMMON_CONTENT = { encryption_status: boldStatusLabel, - install_settings_dns: upstream, + install_settings_dns: upstreamString, elapsed: formattedElapsedMs, response_code: status, ...(service_name @@ -90,8 +94,9 @@ const ResponseCell = ({ const detailedInfo = getDetailedInfo(reason); - return
- + -
-
{statusLabel}
- {isDetailed &&
{detailedInfo}
} + /> +
+
{statusLabel}
+ {isDetailed &&
{detailedInfo}
} +
-
; + ); }; ResponseCell.propTypes = { @@ -117,6 +123,7 @@ ResponseCell.propTypes = { response: propTypes.array.isRequired, status: propTypes.string.isRequired, upstream: propTypes.string.isRequired, + cached: propTypes.bool.isRequired, rules: propTypes.arrayOf(propTypes.shape({ text: propTypes.string.isRequired, filter_list_id: propTypes.number.isRequired, diff --git a/client/src/components/Logs/Cells/index.js b/client/src/components/Logs/Cells/index.js index df36f1d2468..2287f8d19c8 100644 --- a/client/src/components/Logs/Cells/index.js +++ b/client/src/components/Logs/Cells/index.js @@ -76,6 +76,7 @@ const Row = memo(({ originalResponse, status, service_name, + cached, } = rowProps; const hasTracker = !!tracker; @@ -116,6 +117,9 @@ const Row = memo(({ const blockingForClientKey = isFiltered ? 'unblock_for_this_client_only' : 'block_for_this_client_only'; const clientNameBlockingFor = getBlockingClientName(clients, client); + const upstreamString = cached + ? t('served_from_cache', { value: upstream, i: }) + : upstream; const onBlockingForClientClick = () => { dispatch(toggleBlockingForClient(buttonType, domain, clientNameBlockingFor)); @@ -175,7 +179,7 @@ const Row = memo(({ className="link--green">{sourceData.name} , response_details: 'title', - install_settings_dns: upstream, + install_settings_dns: upstreamString, elapsed: formattedElapsedMs, ...(rules.length > 0 && { rule_label: getRulesToFilterList(rules, filters, whitelistFilters) } @@ -230,6 +234,7 @@ Row.propTypes = { time: propTypes.string.isRequired, tracker: propTypes.object, upstream: propTypes.string.isRequired, + cached: propTypes.bool.isRequired, type: propTypes.string.isRequired, client_proto: propTypes.string.isRequired, client_id: propTypes.string, diff --git a/client/src/helpers/helpers.js b/client/src/helpers/helpers.js index 4c9803ae3a5..8f4344eeef7 100644 --- a/client/src/helpers/helpers.js +++ b/client/src/helpers/helpers.js @@ -75,6 +75,7 @@ export const normalizeLogs = (logs) => logs.map((log) => { service_name, original_answer, upstream, + cached, } = log; const { name: domain, unicode_name: unicodeName, type } = question; @@ -116,6 +117,7 @@ export const normalizeLogs = (logs) => logs.map((log) => { answer_dnssec, elapsedMs, upstream, + cached, }; }); diff --git a/client/src/helpers/validators.js b/client/src/helpers/validators.js index 2ac98a6be1f..2ebf6a30ee5 100644 --- a/client/src/helpers/validators.js +++ b/client/src/helpers/validators.js @@ -230,17 +230,6 @@ export const validateMac = (value) => { return undefined; }; -/** - * @param value {number} - * @returns {boolean|*} - */ -export const validateBiggerOrEqualZeroValue = (value) => { - if (value < 0) { - return 'form_error_negative'; - } - return false; -}; - /** * @param value {number} * @returns {undefined|string} diff --git a/go.mod b/go.mod index 7c032047127..9d4593ca3b5 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/AdguardTeam/AdGuardHome go 1.16 require ( - github.com/AdguardTeam/dnsproxy v0.39.9 - github.com/AdguardTeam/golibs v0.10.2 + github.com/AdguardTeam/dnsproxy v0.39.12 + github.com/AdguardTeam/golibs v0.10.3 github.com/AdguardTeam/urlfilter v0.15.0 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.2.2 diff --git a/go.sum b/go.sum index 6db8fa82606..969ec1918e7 100644 --- a/go.sum +++ b/go.sum @@ -9,13 +9,13 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AdguardTeam/dhcp v0.0.0-20210519141215-51808c73c0bf h1:gc042VRSIRSUzZ+Px6xQCRWNJZTaPkomisDfUZmoFNk= github.com/AdguardTeam/dhcp v0.0.0-20210519141215-51808c73c0bf/go.mod h1:TKl4jN3Voofo4UJIicyNhWGp/nlQqQkFxmwIFTvBkKI= -github.com/AdguardTeam/dnsproxy v0.39.9 h1:lH4lKA7KHKFJZgzlij1YAVX6v7eIQpUFpYh9qV+WfGw= -github.com/AdguardTeam/dnsproxy v0.39.9/go.mod h1:eDpJKAdkHORRwAedjuERv+7SWlcz4cn+5uwrbUAWHRY= +github.com/AdguardTeam/dnsproxy v0.39.12 h1:BxAfdQLGnu0rqhD23K5nNw09sKQeqpT2AkFEE78WOqU= +github.com/AdguardTeam/dnsproxy v0.39.12/go.mod h1:g7zjF1TWpKNeDVh6h3YrjQN8565zsWRd7zo++C/935c= github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/golibs v0.4.2/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/golibs v0.9.2/go.mod h1:fCAMwPBJ8S7YMYbTWvYS+eeTLblP5E04IDtNAo7y7IY= -github.com/AdguardTeam/golibs v0.10.2 h1:TAwnS4Y49sSUa4UX1yz/MWNGbIlXHqafrWr9MxdIh9A= -github.com/AdguardTeam/golibs v0.10.2/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw= +github.com/AdguardTeam/golibs v0.10.3 h1:FBgk17zf35ESVWQKIqEUiqqB2bDaCBC8X5vMU760yB4= +github.com/AdguardTeam/golibs v0.10.3/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw= github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU= github.com/AdguardTeam/urlfilter v0.15.0 h1:K3WWZE0K5nPTHe2l+TRXDFpYWJJnvkHdlWidt6NQUTk= github.com/AdguardTeam/urlfilter v0.15.0/go.mod h1:EwXwrYhowP7bedqmOrmKKmQtpBYFyDNEBFQ+lxdUgQU= @@ -64,7 +64,6 @@ github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020 h1:mdi6AbCEoKCA1xKCmp7UtRB5fvGFlP92PvlhxgdvXEw= github.com/go-ping/ping v0.0.0-20210506233800-ff8be3320020/go.mod h1:KmHOjTUmJh/l04ukqPoBWPEZr9jwN05h5NXQl5C+DyY= -github.com/go-test/deep v1.0.5 h1:AKODKU3pDH1RzZzm6YZu77YWtEAq6uh1rLIAQlay2qc= github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/internal/dnsforward/stats.go b/internal/dnsforward/stats.go index abdb519e747..67af8ff9c3e 100644 --- a/internal/dnsforward/stats.go +++ b/internal/dnsforward/stats.go @@ -27,11 +27,12 @@ func (s *Server) processQueryLogsAndStats(ctx *dnsContext) (rc resultCode) { shouldLog = false } + ip, _ := netutil.IPAndPortFromAddr(pctx.Addr) + ip = netutil.CloneIP(ip) + s.serverLock.RLock() defer s.serverLock.RUnlock() - ip, _ := netutil.IPAndPortFromAddr(pctx.Addr) - ip = netutil.CloneIP(ip) s.anonymizer.Load()(ip) log.Debug("client ip: %s", ip) @@ -60,12 +61,14 @@ func (s *Server) processQueryLogsAndStats(ctx *dnsContext) (rc resultCode) { case proxy.ProtoDNSCrypt: p.ClientProto = querylog.ClientProtoDNSCrypt default: - // Consider this a plain DNS-over-UDP or DNS-over-TCP - // request. + // Consider this a plain DNS-over-UDP or DNS-over-TCP request. } if pctx.Upstream != nil { p.Upstream = pctx.Upstream.Address() + } else if cachedUps := pctx.CachedUpstreamAddr; cachedUps != "" { + p.Upstream = pctx.CachedUpstreamAddr + p.Cached = true } s.queryLog.Add(p) diff --git a/internal/querylog/decode.go b/internal/querylog/decode.go index d23fc526dd2..65c8ada1334 100644 --- a/internal/querylog/decode.go +++ b/internal/querylog/decode.go @@ -44,8 +44,10 @@ var logEntryHandlers = map[string]logEntryHandler{ if !ok { return nil } + var err error ent.Time, err = time.Parse(time.RFC3339, v) + return err }, "QH": func(t json.Token, ent *logEntry) error { @@ -69,7 +71,9 @@ var logEntryHandlers = map[string]logEntryHandler{ if !ok { return nil } + ent.QClass = v + return nil }, "CP": func(t json.Token, ent *logEntry) error { @@ -77,8 +81,10 @@ var logEntryHandlers = map[string]logEntryHandler{ if !ok { return nil } + var err error ent.ClientProto, err = NewClientProto(v) + return err }, "Answer": func(t json.Token, ent *logEntry) error { @@ -86,8 +92,10 @@ var logEntryHandlers = map[string]logEntryHandler{ if !ok { return nil } + var err error ent.Answer, err = base64.StdEncoding.DecodeString(v) + return err }, "OrigAnswer": func(t json.Token, ent *logEntry) error { @@ -95,16 +103,30 @@ var logEntryHandlers = map[string]logEntryHandler{ if !ok { return nil } + var err error ent.OrigAnswer, err = base64.StdEncoding.DecodeString(v) + return err }, + "Cached": func(t json.Token, ent *logEntry) error { + v, ok := t.(bool) + if !ok { + return nil + } + + ent.Cached = v + + return nil + }, "Upstream": func(t json.Token, ent *logEntry) error { v, ok := t.(string) if !ok { return nil } + ent.Upstream = v + return nil }, "Elapsed": func(t json.Token, ent *logEntry) error { @@ -112,11 +134,14 @@ var logEntryHandlers = map[string]logEntryHandler{ if !ok { return nil } + i, err := v.Int64() if err != nil { return err } + ent.Elapsed = time.Duration(i) + return nil }, } diff --git a/internal/querylog/decode_test.go b/internal/querylog/decode_test.go index d57b24f0208..fe4e085e65f 100644 --- a/internal/querylog/decode_test.go +++ b/internal/querylog/decode_test.go @@ -33,6 +33,7 @@ func TestDecodeLogEntry(t *testing.T) { `"QC":"IN",` + `"CP":"",` + `"Answer":"` + ansStr + `",` + + `"Cached":true,` + `"Result":{` + `"IsFiltered":true,` + `"Reason":3,` + @@ -42,6 +43,7 @@ func TestDecodeLogEntry(t *testing.T) { `"CanonName":"example.com",` + `"ServiceName":"example.org",` + `"DNSRewriteResult":{"RCode":0,"Response":{"1":["127.0.0.2"]}}},` + + `"Upstream":"https://some.upstream",` + `"Elapsed":837429}` ans, err := base64.StdEncoding.DecodeString(ansStr) @@ -56,6 +58,7 @@ func TestDecodeLogEntry(t *testing.T) { ClientID: "cli42", ClientProto: "", Answer: ans, + Cached: true, Result: filtering.Result{ IsFiltered: true, Reason: filtering.FilteredBlockList, @@ -78,7 +81,8 @@ func TestDecodeLogEntry(t *testing.T) { }, }, }, - Elapsed: 837429, + Upstream: "https://some.upstream", + Elapsed: 837429, } got := &logEntry{} diff --git a/internal/querylog/json.go b/internal/querylog/json.go index 23953f80d2a..6f59f1d5fb5 100644 --- a/internal/querylog/json.go +++ b/internal/querylog/json.go @@ -74,6 +74,7 @@ func (l *queryLog) entryToJSON(entry *logEntry, anonFunc aghnet.IPMutFunc) (json "time": entry.Time.Format(time.RFC3339Nano), "client": eip, "client_proto": entry.ClientProto, + "cached": entry.Cached, "upstream": entry.Upstream, "question": question, } diff --git a/internal/querylog/qlog.go b/internal/querylog/qlog.go index 48f0d351477..f04eecb5286 100644 --- a/internal/querylog/qlog.go +++ b/internal/querylog/qlog.go @@ -87,10 +87,11 @@ type logEntry struct { Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net OrigAnswer []byte `json:",omitempty"` + Cached bool `json:",omitempty"` Result filtering.Result Elapsed time.Duration - Upstream string `json:",omitempty"` // if empty, means it was cached + Upstream string `json:",omitempty"` } func (l *queryLog) Start() { @@ -171,6 +172,7 @@ func (l *queryLog) Add(params AddParams) { Result: *params.Result, Elapsed: params.Elapsed, Upstream: params.Upstream, + Cached: params.Cached, ClientID: params.ClientID, ClientProto: params.ClientProto, } diff --git a/internal/querylog/querylog.go b/internal/querylog/querylog.go index 58b5a8e0418..0b39c01e51e 100644 --- a/internal/querylog/querylog.go +++ b/internal/querylog/querylog.go @@ -73,16 +73,24 @@ type Config struct { Anonymizer *aghnet.IPMut } -// AddParams - parameters for Add() +// AddParams is the parameters for adding an entry. type AddParams struct { - Question *dns.Msg - Answer *dns.Msg // The response we sent to the client (optional) - OrigAnswer *dns.Msg // The response from an upstream server (optional) - Result *filtering.Result // Filtering result (optional) - Elapsed time.Duration // Time spent for processing the request - ClientID string - ClientIP net.IP - Upstream string // Upstream server URL + Question *dns.Msg + // Answer is the response which is sent to the client, if any. + Answer *dns.Msg + // OrigAnswer is the response from an upstream server. It's only set if the + // answer has been modified by filtering. + OrigAnswer *dns.Msg + // Cached indicates if the response is served from cache. + Cached bool + // Result is the filtering result (optional). + Result *filtering.Result + // Elapsed is the time spent for processing the request. + Elapsed time.Duration + ClientID string + ClientIP net.IP + // Upstream is the URL of the upstream DNS server. + Upstream string ClientProto ClientProto } diff --git a/openapi/CHANGELOG.md b/openapi/CHANGELOG.md index 7fceb4180df..558490f2f39 100644 --- a/openapi/CHANGELOG.md +++ b/openapi/CHANGELOG.md @@ -4,6 +4,11 @@ ## v0.107: API changes +## The new field `"cached"` in `QueryLogItem` + +* The new field `"cached"` in `GET /control/querylog` is true if the response is + served from cache instead of being resolved by an upstream server. + ### New constant values for `filter_list_id` field in `ResultRule` * Value of `0` is now used for custom filtering rules list. diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index aa2d5aa3b18..ea5aa05b4e0 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -1868,6 +1868,10 @@ 'description': 'Answer from upstream server (optional)' 'items': '$ref': '#/components/schemas/DnsAnswer' + 'cached': + 'type': 'boolean' + 'description': > + Defines if the response has been served from cache. 'upstream': 'type': 'string' 'description': >