Merge branch 'master' of github.com:spiffe/spire into jwt-key-type-re…

…based

.circleci/config.yml

@@ -102,8 +102,8 @@ jobs:
     steps:
       - setup_remote_docker: {}
       - checkout
-      - run: make images
-      - run: docker save spire-server:latest-local spire-agent:latest-local k8s-workload-registrar:latest-local oidc-discovery-provider:latest-local | gzip > images.tar.gz
+      - run: make images scratch-images
+      - run: docker save spire-server:latest-local spire-agent:latest-local k8s-workload-registrar:latest-local oidc-discovery-provider:latest-local spire-server-scratch:latest-local spire-agent-scratch:latest-local k8s-workload-registrar-scratch:latest-local | gzip > images.tar.gz
       - persist_to_workspace:
           root: ./
           paths:
@@ -113,7 +113,8 @@ jobs:
   integration-test:
     # Run in the machine executor since we have to execute a bunch of
     # docker containers to do the tests
-    machine: true
+    machine:
+      image: ubuntu-2004:202010-01
     parallelism: 4
     resource_class: medium
     environment:
@@ -130,9 +131,12 @@ jobs:
       - checkout
       - run:
           name: Execute integration test suites
-          working_directory: test/integration/
           command: |
-            circleci tests glob "suites/*" | circleci tests split | xargs ./test.sh
+            # CircleCI is silly and doesn't provide this incredibly helpful
+            # environment variable. Requests for it go back years. For shame.
+            CIRCLE_TARGET_BRANCH=$(.circleci/determine-target-branch.sh)
+            cd test/integration
+            circleci tests glob "suites/*" | circleci tests split | CICD_TARGET_BRANCH="${CIRCLE_TARGET_BRANCH}" xargs ./test.sh
 
   # Publish "unstable" docker images
   publish-unstable-images:

.circleci/determine-target-branch.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# The intent of this script, when invoked by CircleCI is to mimic the behavior
+# the TRAVIS_BRANCH environment variable to determine the target branch of
+# the build.
+#
+# Namely, it is set to a value according to the following rules:
+# - for push builds, or builds not triggered by a pull request, this is the name of the branch.
+# - for builds triggered by a pull request this is the name of the branch targeted by the pull request.
+# - for builds triggered by a tag, this is the same as the name of the tag (CIRCLECI_TAG).
+
+if [ -n "${CIRCLE_PR_NUMBER}" ]; then
+    curl -s https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'
+elif [ -n "${CIRCLE_TAG}" ]; then
+    echo "${CIRCLE_TAG}"
+else
+    echo "${CIRCLE_BRANCH}"
+fi

.travis.yml

@@ -8,6 +8,16 @@ language: go
 go:
   - 1.15.5
 
+git:
+  # The upgrade integration test inspects the version tags. The default depth
+  # (50) is sometimes not enough if there have been many commits in between
+  # releases. This causes the integration test to fail to locate the previous
+  # version tag. Bumping this to 100 for now. If we encounter the failure again
+  # we may consider just cloning the whole repo. Alternatively, we can run `git
+  # fetch --tags` in the integration test job below but that is effectively the
+  # same thing.
+  depth: 100
+
 cache:
   directories:
     # cache build tools
@@ -92,7 +102,7 @@ jobs:
       script:
         - make images
         - make scratch-images
-        - make integration
+        - CICD_TARGET_BRANCH="$TRAVIS_BRANCH" make integration
         - .travis/publish-images.sh
 
     - stage: nightly integration tests
@@ -102,7 +112,7 @@ jobs:
       script:
         - make images
         - make scratch-images
-        - make integration
+        - CICD_TARGET_BRANCH="$TRAVIS_BRANCH" make integration
 
 notifications:
   email:

CHANGELOG.md

@@ -1,5 +1,54 @@
 # Changelog
 
+## [0.12.0] - 2020-12-17
+
+### Added
+- Debug endpoints (#1792)
+- Agent support for SDS v3 API (#1906)
+- Improved metrics handling (#1885, #1925, #1932)
+- Significantly improved performance related to performing agent authorization lookups (#1859, #1896, #1943, #1944, #1956)
+- Database indexes to attested node columns (#1912)
+- Support for configuring Vault roles, namespaces, and re-authentication to the Vault UpstreamAuthority plugin (#1871, #1981)
+- Support for non-renewable Vault tokens to the Vault UpstreamAuthority plugin (#1965)
+- Delete mode for federated bundles to the bundle API (#1897)
+- The CLI now reads JSON from STDIN for entry create/update commands (#1905)
+- Support for multiple CA bundle files in x509pop (#1949)
+- Added `ExpiresAt` to `entry show` output (#1973)
+- Added `k8s_psat:agent_node_ip` selector (#1979)
+
+### Changed
+- The agent now shuts down when it is no longer attested (#1797)
+- Internals now rely on new server APIs (#1849, #1878, #1907, #1908, #1909, #1913, #1947, #1982, #1998, #2001)
+- Workload API now returns a standardized JWKS object (#1904)
+- Log message casing and punctuation are more consistent with project guidelines (#1950, #1952)
+
+### Deprecated
+- The Registration and Node APIs are deprecated, and a warning is logged on use (#1997)
+- The `registration_api` configuration section is deprecated in favor of `server_api` in the k8s-workload-registrar (#2001)
+
+### Removed
+- Removed some superfluous or otherwise unusable metrics and labels (#1881, #1946, #2004)
+
+### Fixed
+- Fixed CLI exit codes when entry create or update fails (#1990)
+- Fixed a bug that could cause external plugins to become orphaned processes after agent/server shutdown (#1962)
+- Fixed handling of the Vault PKI certificate chain (#2012, #2017)
+- Fixed a bug that could cause some gRPC libraries to fail to connect to the server over HTTP/2 (#1968)
+- Fixed Registration API to validate selector syntax (#1919)
+
+### Security
+- JWT-SVIDs that fail validation are no longer logged (#1953)
+
+## [0.11.2] - 2020-10-29
+
+### What's New
+ - Error messages related to a specific class of software bugs are now rate limited (#1901)
+
+### What's Changed
+- Fixed an issue in the Upstream Authority plugin that could result in a delay in the propagation of bundle updates/changes (#1917)
+- Fixed error messages when attestation is disabled (#1899)
+- Fixed some incorrectly-formatted log messages (#1920)
+
 ## [0.11.1] - 2020-09-29
 
 ### What's New

CODEOWNERS

@@ -1,9 +1,5 @@
 * @evan2645 @amartinezfayo @azdagron @APTy @rturner3
 
-#documentation
-/README.md      @evan2645 @amartinezfayo @azdagron @APTy @marcosy @ajessup
-/doc/           @evan2645 @amartinezfayo @azdagron @APTy @marcosy @ajessup
-
 ##########################################
 # Maintainers
 ##########################################
@@ -28,13 +24,6 @@
 # Uber Technologies, Inc
 # @rturner3
 
-##########################################
-# Product Manager
-##########################################
-
-# Andres Vega
-# VMware, Inc
-# @anvega
 
 ##########################################
 # Community Chair

Makefile

@@ -32,6 +32,8 @@ help:
 	@echo "  $(cyan)test$(reset)                                  - run unit tests"
 	@echo "  $(cyan)race-test$(reset)                             - run unit tests with race detection"
 	@echo "  $(cyan)integration$(reset)                           - run integration tests (requires Docker images)"
+	@echo "                                          support 'SUITES' variable for executing specific tests"
+	@echo "                                          e.g. SUITES='suites/join-token suites/k8s' make integration"
 	@echo
 	@echo "$(bold)Build and test:$(reset)"
 	@echo "  $(cyan)all$(reset)                                   - build all SPIRE binaries, lint the code, and run unit tests"
@@ -143,6 +145,7 @@ protos := \
 	proto/spire/types/attestation.proto \
 	proto/spire/types/bundle.proto \
 	proto/spire/types/entry.proto \
+	proto/spire/types/federateswith.proto \
 	proto/spire/types/jointoken.proto \
 	proto/spire/types/jwtsvid.proto \
 	proto/spire/types/selector.proto \
@@ -155,7 +158,6 @@ serviceprotos := \
 	proto/spire/agent/keymanager/keymanager.proto \
 	proto/spire/agent/nodeattestor/nodeattestor.proto \
 	proto/spire/agent/workloadattestor/workloadattestor.proto \
-	proto/spire/api/node/node.proto \
 	proto/spire/api/registration/registration.proto \
 	proto/spire/common/hostservices/metricsservice.proto \
 	proto/spire/common/plugin/plugin.proto \
@@ -354,7 +356,7 @@ else
 endif
 
 integration:
-	$(E)./test/integration/test.sh
+	$(E)./test/integration/test.sh $(SUITES)
 
 #############################################################################
 # Build Artifact

README.md

@@ -33,13 +33,16 @@ SPIRE is hosted by the [Cloud Native Computing Foundation](https://cncf.io) (CNC
 - Check [ADOPTERS.md](./ADOPTERS.md) for a list of production SPIRE adopters, a view of the ecosystem, and use cases.
 - See the [SPIRE Roadmap](https://github.com/spiffe/spire/wiki/Roadmap) for a list of planned features and enhancements.
 - [Join](https://slack.spiffe.io/) the SPIFFE community on Slack. If you have any questions about how SPIRE works, or how to get it up and running, the best places to ask questions are the [SPIFFE Slack channels](https://spiffe.slack.com).
+- Download the free book about SPIFFE and SPIRE, "[Solving the Bottom Turtle](https://spiffe.io/book/)."
 
 ## Integrate with SPIRE
 
 - See [Extend SPIRE](https://spiffe.io/spire/docs/extending/) to learn about the highly extensible SPIRE plugin framework.
 - Client libraries for interacting with the [SPIFFE Workload API](https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE_Workload_API.md) are available in [Go](https://github.com/spiffe/go-spiffe/tree/master/v2), [Java](https://github.com/spiffe/java-spiffe) and [C++](https://github.com/spiffe/c-spiffe) languages. See [SPIFFE Library Usage Examples](https://spiffe.io/spire/try/spiffe-library-usage-examples/) for code samples.
 - SPIRE provides an implementation of the [Envoy](https://envoyproxy.io) [Secret Discovery Service](https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret) (SDS) for use with [Envoy Proxy](https://envoyproxy.io).  SDS can be used to transparently install and rotate TLS certificates and trust bundles in Envoy. See [Using SPIRE with Envoy](https://spiffe.io/spire/docs/envoy/) for more information.
 
+For supported integration versions, see [Supported Integrations](/doc/supported_integrations.md).
+
 ## Contribute to SPIRE
 
 The SPIFFE community maintains the SPIRE project. Information on the various SIGs and relevant standards can be found in

api/workload/v2/workload.go

@@ -20,7 +20,7 @@ import (
 
 const (
 	// DefaultAgentAddress is the default GRPC address to contact the spire agent at.
-	DefaultAgentAddress = "unix:///tmp/agent.sock"
+	DefaultAgentAddress = "unix:///tmp/spire-agent/public/api.sock"
 
 	// EnvVarAgentAddress is the environment variable name where the Workload API address may be configured.
 	EnvVarAgentAddress = "SPIFFE_ENDPOINT_SOCKET"

api/workload/v2/workload_test.go

@@ -127,7 +127,7 @@ func TestStartStop(t *testing.T) {
 func TestGetAgentAddress(t *testing.T) {
 	t.Run("default", func(t *testing.T) {
 		os.Unsetenv(EnvVarAgentAddress)
-		require.Equal(t, "unix:///tmp/agent.sock", GetAgentAddress())
+		require.Equal(t, "unix:///tmp/spire-agent/public/api.sock", GetAgentAddress())
 	})
 	t.Run("env", func(t *testing.T) {
 		os.Setenv(EnvVarAgentAddress, "/foo")

cmd/spire-agent/cli/api/common.go

@@ -74,7 +74,7 @@ func adaptCommand(env *cli.Env, clientsMaker workloadClientMaker, cmd command) *
 
 	fs := flag.NewFlagSet(cmd.name(), flag.ContinueOnError)
 	fs.SetOutput(env.Stderr)
-	fs.StringVar(&a.socketPath, "socketPath", common.DefaultSocketPath, "Path to Workload API socket")
+	fs.StringVar(&a.socketPath, "socketPath", common.DefaultSocketPath, "Path to the SPIRE Agent API socket")
 	fs.Var(&a.timeout, "timeout", "Time to wait for a response")
 	a.cmd.appendFlags(fs)
 	a.flags = fs

cmd/spire-agent/cli/api/watch.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/spiffe/spire/api/workload"
+	"github.com/spiffe/spire/cmd/spire-agent/cli/common"
 )
 
 type WatchConfig struct {
@@ -67,7 +68,7 @@ func (w *WatchCLI) Run(args []string) int {
 func (w *WatchCLI) parseConfig(args []string) error {
 	fs := flag.NewFlagSet("watch", flag.ContinueOnError)
 	c := &WatchConfig{}
-	fs.StringVar(&c.socketPath, "socketPath", "/tmp/agent.sock", "Path to the Workload API socket")
+	fs.StringVar(&c.socketPath, "socketPath", common.DefaultSocketPath, "Path to the Workload API socket")
 
 	w.config = c
 	return fs.Parse(args)

cmd/spire-agent/cli/common/defaults.go

@@ -2,5 +2,5 @@ package common
 
 const (
 	// DefaultSocketPath is the SPIRE agent's default socket path
-	DefaultSocketPath = "/tmp/agent.sock"
+	DefaultSocketPath = "/tmp/spire-agent/public/api.sock"
 )

cmd/spire-agent/cli/healthcheck/healthcheck.go

@@ -1,17 +1,17 @@
 package healthcheck
 
 import (
+	"context"
 	"errors"
 	"flag"
+	"fmt"
 	"net"
-	"time"
 
 	"github.com/mitchellh/cli"
-	api_workload "github.com/spiffe/spire/api/workload"
+	"github.com/spiffe/spire/api/workload/dial"
 	"github.com/spiffe/spire/cmd/spire-agent/cli/common"
 	common_cli "github.com/spiffe/spire/pkg/common/cli"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/health/grpc_health_v1"
 )
 
 func NewHealthCheckCommand() cli.Command {
@@ -20,16 +20,14 @@ func NewHealthCheckCommand() cli.Command {
 
 func newHealthCheckCommand(env *common_cli.Env) *healthCheckCommand {
 	return &healthCheckCommand{
-		env:     env,
-		timeout: common_cli.DurationFlag(time.Second * 5),
+		env: env,
 	}
 }
 
 type healthCheckCommand struct {
 	env *common_cli.Env
 
 	socketPath string
-	timeout    common_cli.DurationFlag
 	shallow    bool
 	verbose    bool
 }
@@ -49,9 +47,12 @@ func (c *healthCheckCommand) Run(args []string) int {
 		return 1
 	}
 	if err := c.run(); err != nil {
-		// Ignore error since a failure to write to stderr cannot very well
-		// be reported
-		_ = c.env.ErrPrintln(err)
+		// Ignore error since a failure to write to stderr cannot very well be
+		// reported
+		_ = c.env.ErrPrintf("Agent is unhealthy: %v\n", err)
+		return 1
+	}
+	if err := c.env.Println("Agent is healthy."); err != nil {
 		return 1
 	}
 	return 0
@@ -60,51 +61,40 @@ func (c *healthCheckCommand) Run(args []string) int {
 func (c *healthCheckCommand) parseFlags(args []string) error {
 	fs := flag.NewFlagSet("health", flag.ContinueOnError)
 	fs.SetOutput(c.env.Stderr)
-	fs.StringVar(&c.socketPath, "socketPath", common.DefaultSocketPath, "Path to Workload API socket")
+	fs.StringVar(&c.socketPath, "socketPath", common.DefaultSocketPath, "Path to the SPIRE Agent API socket")
 	fs.BoolVar(&c.shallow, "shallow", false, "Perform a less stringent health check")
 	fs.BoolVar(&c.verbose, "verbose", false, "Print verbose information")
 	return fs.Parse(args)
 }
 
 func (c *healthCheckCommand) run() error {
-	addr := &net.UnixAddr{
-		Name: c.socketPath,
-		Net:  "unix",
-	}
-
 	if c.verbose {
-		c.env.Printf("Contacting Workload API...\n")
+		c.env.Printf("Checking agent health...\n")
 	}
 
-	client := api_workload.NewX509Client(&api_workload.X509ClientConfig{
-		Addr:        addr,
-		FailOnError: true,
+	conn, err := dial.Dial(context.Background(), &net.UnixAddr{
+		Name: c.socketPath,
+		Net:  "unix",
 	})
-	defer client.Stop()
-
-	errCh := make(chan error, 1)
-	go func() {
-		errCh <- client.Start()
-	}()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
 
-	select {
-	case err := <-errCh:
+	healthClient := grpc_health_v1.NewHealthClient(conn)
+	resp, err := healthClient.Check(context.Background(), &grpc_health_v1.HealthCheckRequest{})
+	if err != nil {
 		if c.verbose {
-			c.env.Printf("Workload API returned %s\n", err)
-		}
-		if status.Code(err) == codes.Unavailable {
-			return errors.New("Agent is unavailable.") //nolint: golint // error is (ab)used for CLI output
-		}
-	case <-client.UpdateChan():
-		if c.verbose {
-			if err := c.env.Println("SVID received over Workload API."); err != nil {
-				return err
-			}
+			// Ignore error since a failure to write to stderr cannot very well
+			// be reported
+			_ = c.env.ErrPrintf("Failed to check health: %v\n", err)
 		}
+		return errors.New("unable to determine health")
 	}
 
-	if err := c.env.Println("Agent is healthy."); err != nil {
-		return err
+	if resp.Status != grpc_health_v1.HealthCheckResponse_SERVING {
+		return fmt.Errorf("agent returned status %q", resp.Status)
 	}
+
 	return nil
 }