Merge branch 'main' into gh-17320-update-metrics-doc

.changelog/15654.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+cli: Adds new command - `consul services export` - for exporting a service to a peer or partition
+```

.changelog/16845.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+systemd: set service type to notify.
+```

.changelog/17055.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+gateways: Fix an bug where targeting a virtual service defined by a service-resolver was broken for HTTPRoutes.
+```

.changelog/17525.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+http: accept query parameters `datacenter`, `ap` (enterprise-only), and `namespace` (enterprise-only). Both short-hand and long-hand forms of these query params are now supported via the HTTP API (dc/datacenter, ap/partition, ns/namespace).
+```

.changelog/17566.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+xds: Fixed a bug where modifying ACLs on a token being actively used for an xDS connection caused all xDS updates to fail.
+```

.changelog/4633.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+server: **(Enterprise Only)** added server side RPC requests IP based read/write rate-limiter.
+```

.changelog/5102.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+server: **(Enterprise Only)** allow automatic license utilization reporting.
+```

.gitignore

@@ -66,3 +66,5 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+/go.work
+/go.work.sum

.golangci.yml

@@ -88,6 +88,9 @@ linters-settings:
         - github.com/hashicorp/go-msgpack:
             recommendations:
               - github.com/hashicorp/consul-net-rpc/go-msgpack
+        - github.com/golang/protobuf:
+            recommendations:
+              - google.golang.org/protobuf
 
   depguard:
     list-type: denylist
@@ -101,7 +104,9 @@ linters-settings:
     # Default: []
     packages-with-error-message:
       - net/rpc: "only use forked copy in github.com/hashicorp/consul-net-rpc/net/rpc"
+      - github.com/golang/protobuf: "only use google.golang.org/protobuf"
 
 run:
   timeout: 10m
   concurrency: 4
+  skip-dirs-use-default: false

.release/linux/package/usr/lib/systemd/system/consul.service

@@ -6,6 +6,7 @@ After=network-online.target
 ConditionFileNotEmpty=/etc/consul.d/consul.hcl
 
 [Service]
+Type=notify
 EnvironmentFile=-/etc/consul.d/consul.env
 User=consul
 Group=consul

CHANGELOG.md

@@ -1,3 +1,61 @@
+## 1.15.3 (June 1, 2023)
+
+BREAKING CHANGES:
+
+* extensions: The Lua extension now targets local proxy listeners for the configured service's upstreams, rather than remote downstream listeners for the configured service, when ListenerType is set to outbound in extension configuration. See [CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816) changelog entry for more details. [[GH-17415](https://github.com/hashicorp/consul/issues/17415)]
+
+SECURITY:
+
+* Update to UBI base image to 9.2. [[GH-17513](https://github.com/hashicorp/consul/issues/17513)]
+* Upgrade golang.org/x/net to address [CVE-2022-41723](https://nvd.nist.gov/vuln/detail/CVE-2022-41723) [[GH-16754](https://github.com/hashicorp/consul/issues/16754)]
+* Upgrade to use Go 1.20.4.
+This resolves vulnerabilities [CVE-2023-24537](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`go/scanner`),
+[CVE-2023-24538](https://github.com/advisories/GHSA-v4m2-x4rp-hv22)(`html/template`),
+[CVE-2023-24534](https://github.com/advisories/GHSA-8v5j-pwr7-w5f8)(`net/textproto`) and
+[CVE-2023-24536](https://github.com/advisories/GHSA-9f7g-gqwh-jpf5)(`mime/multipart`).
+Also, `golang.org/x/net` has been updated to v0.7.0 to resolve CVEs [CVE-2022-41721
+](https://github.com/advisories/GHSA-fxg5-wq6x-vr4w
+), [CVE-2022-27664](https://github.com/advisories/GHSA-69cg-p879-7622) and [CVE-2022-41723
+](https://github.com/advisories/GHSA-vvpx-j8f3-3w6h
+.) [[GH-17240](https://github.com/hashicorp/consul/issues/17240)]
+* extensions: Disable remote downstream proxy patching by Envoy Extensions other than AWS Lambda. Previously, an operator with service:write ACL permissions for an upstream service could modify Envoy proxy config for downstream services without equivalent permissions for those services. This issue only impacts the Lua extension. [[CVE-2023-2816](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-2816)] [[GH-17415](https://github.com/hashicorp/consul/issues/17415)]
+
+FEATURES:
+
+* hcp: Add new metrics sink to collect, aggregate and export server metrics to HCP in OTEL format. [[GH-17460](https://github.com/hashicorp/consul/issues/17460)]
+
+IMPROVEMENTS:
+
+* Fixes a performance issue in Raft where commit latency can increase by 100x or more when under heavy load. For more details see https://github.com/hashicorp/raft/pull/541. [[GH-17081](https://github.com/hashicorp/consul/issues/17081)]
+* agent: add a configurable maximimum age (default: 7 days) to prevent servers re-joining a cluster with stale data [[GH-17171](https://github.com/hashicorp/consul/issues/17171)]
+* agent: add new metrics to track cpu disk and memory usage for server hosts (defaults to: enabled) [[GH-17038](https://github.com/hashicorp/consul/issues/17038)]
+* connect: update supported envoy versions to 1.22.11, 1.23.8, 1.24.6, 1.25.4 [[GH-16889](https://github.com/hashicorp/consul/issues/16889)]
+* envoy: add `MaxEjectionPercent` and `BaseEjectionTime` to passive health check configs. [[GH-15979](https://github.com/hashicorp/consul/issues/15979)]
+* hcp: Add support for linking existing Consul clusters to HCP management plane. [[GH-16916](https://github.com/hashicorp/consul/issues/16916)]
+* logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot` [[GH-17236](https://github.com/hashicorp/consul/issues/17236)]
+* peering: allow re-establishing terminated peering from new token without deleting existing peering first. [[GH-16776](https://github.com/hashicorp/consul/issues/16776)]
+* peering: gRPC queries for TrustBundleList, TrustBundleRead, PeeringList, and PeeringRead now support blocking semantics,
+ reducing network and CPU demand.
+ The HTTP APIs for Peering List and Read have been updated to support blocking. [[GH-17426](https://github.com/hashicorp/consul/issues/17426)]
+* raft: Remove expensive reflection from raft/mesh hot path [[GH-16552](https://github.com/hashicorp/consul/issues/16552)]
+* xds: rename envoy_hcp_metrics_bind_socket_dir to envoy_telemetry_collector_bind_socket_dir to remove HCP naming references. [[GH-17327](https://github.com/hashicorp/consul/issues/17327)]
+
+BUG FIXES:
+
+* Fix an bug where decoding some Config structs with unset pointer fields could fail with `reflect: call of reflect.Value.Type on zero Value`. [[GH-17048](https://github.com/hashicorp/consul/issues/17048)]
+* acl: **(Enterprise only)** Check permissions in correct partition/namespace when resolving service in non-default partition/namespace
+* acl: Fix an issue where the anonymous token was synthesized in non-primary datacenters which could cause permission errors when federating clusters with ACL replication enabled. [[GH-17231](https://github.com/hashicorp/consul/issues/17231)]
+* acls: Fix ACL bug that can result in sidecar proxies having incorrect endpoints.
+* connect: Fix multiple inefficient behaviors when querying service health. [[GH-17241](https://github.com/hashicorp/consul/issues/17241)]
+* gateways: Fix an bug where targeting a virtual service defined by a service-resolver was broken for HTTPRoutes. [[GH-17055](https://github.com/hashicorp/consul/issues/17055)]
+* grpc: ensure grpc resolver correctly uses lan/wan addresses on servers [[GH-17270](https://github.com/hashicorp/consul/issues/17270)]
+* namespaces: adjusts the return type from HTTP list API to return the `api` module representation of a namespace.
+This fixes an error with the `consul namespace list` command when a namespace has a deferred deletion timestamp.
+* peering: Fix issue where modifying the list of exported services did not correctly replicate changes for services that exist in a non-default namespace. [[GH-17456](https://github.com/hashicorp/consul/issues/17456)]
+* peering: Fix issue where peer streams could incorrectly deregister services in various scenarios. [[GH-17235](https://github.com/hashicorp/consul/issues/17235)]
+* peering: ensure that merged central configs of peered upstreams for partitioned downstreams work [[GH-17179](https://github.com/hashicorp/consul/issues/17179)]
+* xds: Fix possible panic that can when generating clusters before the root certificates have been fetched. [[GH-17185](https://github.com/hashicorp/consul/issues/17185)]
+
 ## 1.14.7 (May 16, 2023)
 
 SECURITY:

GNUmakefile

@@ -3,6 +3,8 @@
 
 SHELL = bash
 
+GO_MODULES := $(shell find . -name go.mod -exec dirname {} \; | grep -v "proto-gen-rpc-glue/e2e" | sort)
+
 ###
 # These version variables can either be a valid string for "go install <module>@<version>"
 # or the string @DEV to imply use what is currently installed locally.
@@ -249,19 +251,13 @@ cov: other-consul dev-build
 
 test: other-consul dev-build lint test-internal
 
-go-mod-tidy:
-	@echo "--> Running go mod tidy"
-	@cd sdk && go mod tidy
-	@cd api && go mod tidy
-	@go mod tidy
-	@cd test/integration/consul-container && go mod tidy
-	@cd test/integration/connect/envoy/test-sds-server && go mod tidy
-	@cd proto-public && go mod tidy
-	@cd internal/tools/proto-gen-rpc-glue && go mod tidy
-	@cd internal/tools/proto-gen-rpc-glue/e2e && go mod tidy
-	@cd internal/tools/proto-gen-rpc-glue/e2e/consul && go mod tidy
-	@cd internal/tools/protoc-gen-consul-rate-limit && go mod tidy
+.PHONY: go-mod-tidy
+go-mod-tidy: $(foreach mod,$(GO_MODULES),go-mod-tidy/$(mod))
 
+.PHONY: mod-tidy/%
+go-mod-tidy/%:
+	@echo "--> Running go mod tidy ($*)"
+	@cd $* && go mod tidy
 
 test-internal:
 	@echo "--> Running go test"
@@ -292,6 +288,12 @@ test-internal:
 	@grep '^FAIL' test.log || true
 	@if [ "$$(cat exit-code)" == "0" ] ; then echo "PASS" ; exit 0 ; else exit 1 ; fi
 
+test-all: other-consul dev-build lint $(foreach mod,$(GO_MODULES),test-module/$(mod))
+
+test-module/%:
+	@echo "--> Running go test ($*)"
+	cd $* && go test $(GOTEST_FLAGS) -tags '$(GOTAGS)' ./...
+
 test-race:
 	$(MAKE) GOTEST_FLAGS=-race
 
@@ -328,21 +330,26 @@ other-consul:
 		echo "Found other running consul agents. This may affect your tests." ; \
 		exit 1 ; \
 	fi
-
-lint: -lint-main lint-container-test-deps
-
-.PHONY: -lint-main
--lint-main: lint-tools
-	@echo "--> Running golangci-lint"
-	@golangci-lint run --build-tags '$(GOTAGS)' && \
-		(cd api && golangci-lint run --build-tags '$(GOTAGS)') && \
-		(cd sdk && golangci-lint run --build-tags '$(GOTAGS)')
-	@echo "--> Running golangci-lint (container tests)"
-	@cd test/integration/consul-container && golangci-lint run --build-tags '$(GOTAGS)'
-	@echo "--> Running lint-consul-retry"
-	@lint-consul-retry
-	@echo "--> Running enumcover"
-	@enumcover ./...
+
+.PHONY: fmt
+fmt: $(foreach mod,$(GO_MODULES),fmt/$(mod)) 
+
+.PHONY: fmt/%
+fmt/%:
+	@echo "--> Running go fmt ($*)"
+	@cd $* && go fmt ./...
+
+.PHONY: lint
+lint: $(foreach mod,$(GO_MODULES),lint/$(mod)) lint-container-test-deps
+
+.PHONY: lint/%
+lint/%:
+	@echo "--> Running golangci-lint ($*)"
+	@cd $* && GOWORK=off golangci-lint run --build-tags '$(GOTAGS)'
+	@echo "--> Running lint-consul-retry ($*)"
+	@cd $* && GOWORK=off lint-consul-retry
+	@echo "--> Running enumcover ($*)"
+	@cd $* && GOWORK=off enumcover ./...
 
 .PHONY: lint-container-test-deps
 lint-container-test-deps:

agent/agent.go

@@ -597,6 +597,12 @@ func (a *Agent) Start(ctx context.Context) error {
 	// regular and on-demand state synchronizations (anti-entropy).
 	a.sync = ae.NewStateSyncer(a.State, c.AEInterval, a.shutdownCh, a.logger)
 
+	err = validateFIPSConfig(a.config)
+	if err != nil {
+		// Log warning, rather than force breaking
+		a.logger.Warn("FIPS 140-2 Compliance", "issue", err)
+	}
+
 	// create the config for the rpc server/client
 	consulCfg, err := newConsulConfig(a.config, a.logger)
 	if err != nil {
@@ -4565,13 +4571,13 @@ func (a *Agent) persistServerMetadata() {
 
 			f, err := consul.OpenServerMetadata(file)
 			if err != nil {
-				a.logger.Error("failed to open existing server metadata: %w", err)
+				a.logger.Error("failed to open existing server metadata", "error", err)
 				continue
 			}
 
 			if err := consul.WriteServerMetadata(f); err != nil {
 				f.Close()
-				a.logger.Error("failed to write server metadata: %w", err)
+				a.logger.Error("failed to write server metadata", "error", err)
 				continue
 			}
 

agent/agent_endpoint.go

@@ -11,11 +11,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hashicorp/consul/envoyextensions/xdscommon"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-memdb"
 	"github.com/mitchellh/hashstructure"
 
+	"github.com/hashicorp/consul/envoyextensions/xdscommon"
+	"github.com/hashicorp/consul/version"
+
 	"github.com/hashicorp/go-bexpr"
 	"github.com/hashicorp/serf/coordinate"
 	"github.com/hashicorp/serf/serf"
@@ -1683,3 +1685,12 @@ func (s *HTTPHandlers) AgentHost(resp http.ResponseWriter, req *http.Request) (i
 
 	return debug.CollectHostInfo(), nil
 }
+
+// AgentVersion
+//
+// GET /v1/agent/version
+//
+// Retrieves Consul version information.
+func (s *HTTPHandlers) AgentVersion(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	return version.GetBuildInfo(), nil
+}

agent/agent_endpoint_test.go

@@ -21,7 +21,10 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
+
 	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/version"
+
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/serf/serf"
@@ -8081,6 +8084,32 @@ func TestAgent_HostBadACL(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.Code)
 }
 
+func TestAgent_Version(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
+	t.Parallel()
+
+	dc1 := "dc1"
+	a := NewTestAgent(t, `
+		primary_datacenter = "`+dc1+`"
+	`)
+	defer a.Shutdown()
+
+	testrpc.WaitForLeader(t, a.RPC, "dc1")
+	req, _ := http.NewRequest("GET", "/v1/agent/version", nil)
+	// req.Header.Add("X-Consul-Token", "initial-management")
+	resp := httptest.NewRecorder()
+	respRaw, err := a.srv.AgentVersion(resp, req)
+	assert.Nil(t, err)
+	assert.Equal(t, http.StatusOK, resp.Code)
+	assert.NotNil(t, respRaw)
+
+	obj := respRaw.(*version.BuildInfo)
+	assert.NotNil(t, obj.HumanVersion)
+}
+
 // Thie tests that a proxy with an ExposeConfig is returned as expected.
 func TestAgent_Services_ExposeConfig(t *testing.T) {
 	if testing.Short() {

agent/agent_oss.go

@@ -40,6 +40,11 @@ func (a *Agent) reloadEnterprise(conf *config.RuntimeConfig) error {
 func enterpriseConsulConfig(_ *consul.Config, _ *config.RuntimeConfig) {
 }
 
+// validateFIPSConfig is a noop stub for the func defined in agent_ent.go
+func validateFIPSConfig(_ *config.RuntimeConfig) error {
+	return nil
+}
+
 // WriteEvent is a noop stub for the func defined agent_ent.go
 func (a *Agent) WriteEvent(eventType string, payload interface{}) {
 }

agent/consul/discoverychain/gateway.go

@@ -249,6 +249,13 @@ func targetForResolverNode(nodeName string, chains []*structs.CompiledDiscoveryC
 	splitterName := splitterPrefix + strings.TrimPrefix(nodeName, resolverPrefix)
 
 	for _, c := range chains {
+		targetChainPrefix := resolverPrefix + c.ServiceName + "."
+		if strings.HasPrefix(nodeName, targetChainPrefix) && len(c.Nodes) == 1 {
+			// we have a virtual resolver that just maps to another resolver, return
+			// the given node name
+			return c.StartNode
+		}
+
 		for name, node := range c.Nodes {
 			if node.IsSplitter() && strings.HasPrefix(splitterName, name) {
 				return name