Add Kubernetes metadata

We sniff kubernetes pod metadata from /proc/self/cgroup,
and add or override any values specified through the environment
variables KUBERNETES_{NAMESPACE,NODE_NAME,POD_NAME,POD_UID}.

Also, stop checking for "docker" in the cgroup path. The
64-bit hex ID check should be sufficient to avoid false
positives. We only do this check for non-k8s environments.

env_test.go

@@ -186,40 +186,13 @@ func testTracerSanitizeFieldNamesEnv(t *testing.T, envValue, expect string) {
 }
 
 func TestTracerServiceNameEnvSanitizationSpecified(t *testing.T) {
-	testTracerServiceNameSanitization(
-		t, "foo_bar", "ELASTIC_APM_SERVICE_NAME=foo!bar",
-	)
+	_, _, service := getSubprocessMetadata(t, "ELASTIC_APM_SERVICE_NAME=foo!bar")
+	assert.Equal(t, "foo_bar", service.Name)
 }
 
 func TestTracerServiceNameEnvSanitizationExecutableName(t *testing.T) {
-	testTracerServiceNameSanitization(
-		t, "apm_test", // .test -> _test
-	)
-}
-
-func testTracerServiceNameSanitization(t *testing.T, sanitizedServiceName string, env ...string) {
-	if os.Getenv("_INSIDE_TEST") != "1" {
-		cmd := exec.Command(os.Args[0], "-test.run=^"+t.Name()+"$")
-		cmd.Env = append(os.Environ(), "_INSIDE_TEST=1")
-		cmd.Env = append(cmd.Env, env...)
-		output, err := cmd.CombinedOutput()
-		if !assert.NoError(t, err) {
-			t.Logf("output:\n%s", output)
-		}
-		return
-	}
-
-	var transport transporttest.RecorderTransport
-	tracer, _ := apm.NewTracer("", "")
-	tracer.Transport = &transport
-	defer tracer.Close()
-
-	tx := tracer.StartTransaction("name", "type")
-	tx.End()
-	tracer.Flush(nil)
-
-	_, _, service := transport.Metadata()
-	assert.Equal(t, sanitizedServiceName, service.Name)
+	_, _, service := getSubprocessMetadata(t)
+	assert.Equal(t, "apm_test", service.Name) // .test -> _test
 }
 
 func TestTracerCaptureBodyEnv(t *testing.T) {

internal/apmhostutil/container.go

@@ -1,13 +1,17 @@
 package apmhostutil
 
-// ContainerID returns the full container ID in which the process is executing,
-// or an error if the container ID could not be determined.
-func ContainerID() (string, error) {
-	return DockerContainerID()
+import "go.elastic.co/apm/model"
+
+// Container returns information about the container running the process, or an
+// error the container information could not be determined.
+func Container() (*model.Container, error) {
+	return containerInfo()
 }
 
-// DockerContainerID returns the full Docker container ID in which the process
-// is executing, or an error if the container ID could not be determined.
-func DockerContainerID() (string, error) {
-	return dockerContainerID()
+// Kubernetes returns information about the Kubernetes node and pod running
+// the process, or an error if they could not be determined. This information
+// does not include the KUBERNETES_* environment variables that can be set via
+// the Downward API.
+func Kubernetes() (*model.Kubernetes, error) {
+	return kubernetesInfo()
 }

internal/apmhostutil/container_linux.go

@@ -0,0 +1,132 @@
+// +build linux
+
+package apmhostutil
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"sync"
+
+	"go.elastic.co/apm/model"
+)
+
+const (
+	systemdScopeSuffix = ".scope"
+)
+
+var (
+	cgroupContainerInfoOnce  sync.Once
+	cgroupContainerInfoError error
+	kubernetes               *model.Kubernetes
+	container                *model.Container
+
+	kubepodsRegexp = regexp.MustCompile(
+		"" +
+			`(?:^/kubepods/[^/]+/pod([^/]+)/$)|` +
+			`(?:^/kubepods\.slice/kubepods-[^/]+\.slice/kubepods-[^/]+-pod([^/]+)\.slice/$)`,
+	)
+
+	containerIDRegexp = regexp.MustCompile("^[[:xdigit:]]{64}$")
+)
+
+func containerInfo() (*model.Container, error) {
+	container, _, err := cgroupContainerInfo()
+	return container, err
+}
+
+func kubernetesInfo() (*model.Kubernetes, error) {
+	_, kubernetes, err := cgroupContainerInfo()
+	if err == nil && kubernetes == nil {
+		return nil, errors.New("could not determine kubernetes info")
+	}
+	return kubernetes, err
+}
+
+func cgroupContainerInfo() (*model.Container, *model.Kubernetes, error) {
+	cgroupContainerInfoOnce.Do(func() {
+		cgroupContainerInfoError = func() error {
+			f, err := os.Open("/proc/self/cgroup")
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			c, k, err := readCgroupContainerInfo(f)
+			if err != nil {
+				return err
+			}
+			if c == nil {
+				return errors.New("could not determine container info")
+			}
+			container = c
+			kubernetes = k
+			return nil
+		}()
+	})
+	return container, kubernetes, cgroupContainerInfoError
+}
+
+func readCgroupContainerInfo(r io.Reader) (*model.Container, *model.Kubernetes, error) {
+	var container *model.Container
+	var kubernetes *model.Kubernetes
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		fields := strings.SplitN(s.Text(), ":", 3)
+		if len(fields) != 3 {
+			continue
+		}
+		cgroupPath := fields[2]
+
+		// Depending on the filesystem driver used for cgroup
+		// management, the paths in /proc/pid/cgroup will have
+		// one of the following formats in a Docker container:
+		//
+		//   systemd: /system.slice/docker-<container-ID>.scope
+		//   cgroupfs: /docker/<container-ID>
+		//
+		// In a Kubernetes pod, the cgroup path will look like:
+		//
+		//   systemd: /kubepods.slice/kubepods-<QoS-class>.slice/kubepods-<QoS-class>-pod<pod-UID>.slice/<container-iD>.scope
+		//   cgroupfs: /kubepods/<QoS-class>/pod<pod-UID>/<container-iD>
+		//
+		dir, id := path.Split(cgroupPath)
+		if strings.HasSuffix(id, systemdScopeSuffix) {
+			id = id[:len(id)-len(systemdScopeSuffix)]
+			if dash := strings.IndexRune(id, '-'); dash != -1 {
+				id = id[dash+1:]
+			}
+		}
+		if match := kubepodsRegexp.FindStringSubmatch(dir); match != nil {
+			// By default, Kubernetes will set the hostname of
+			// the pod containers to the pod name. Users that
+			// override the name should use the Downard API to
+			// override the pod name.
+			hostname, _ := os.Hostname()
+			uid := match[1]
+			if uid == "" {
+				uid = match[2]
+			}
+			kubernetes = &model.Kubernetes{
+				Pod: &model.KubernetesPod{
+					Name: hostname,
+					UID:  uid,
+				},
+			}
+			// We don't check the contents of the last path segment
+			// when we've matched "^/kubepods"; we assume that it is
+			// a valid container ID.
+			container = &model.Container{ID: id}
+		} else if containerIDRegexp.MatchString(id) {
+			container = &model.Container{ID: id}
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, nil, err
+	}
+	return container, kubernetes, nil
+}

internal/apmhostutil/container_linux_test.go

@@ -0,0 +1,119 @@
+package apmhostutil
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"go.elastic.co/apm/model"
+)
+
+func TestCgroupContainerInfoDocker(t *testing.T) {
+	container, kubernetes, err := readCgroupContainerInfo(strings.NewReader(`
+12:devices:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+11:hugetlb:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+10:memory:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+9:freezer:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+8:perf_event:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+7:blkio:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+6:pids:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+5:rdma:/
+4:cpuset:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+3:net_cls,net_prio:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+2:cpu,cpuacct:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+1:name=systemd:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76
+0::/system.slice/docker.service`[1:]))
+
+	assert.NoError(t, err)
+	assert.Nil(t, kubernetes)
+	assert.Equal(t, &model.Container{ID: "051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76"}, container)
+}
+
+func TestCgroupContainerInfoECS(t *testing.T) {
+	container, kubernetes, err := readCgroupContainerInfo(strings.NewReader(`
+3:cpuacct:/ecs/eb9d3d0c-8936-42d7-80d8-f82b2f1a629e/7e9139716d9e5d762d22f9f877b87d1be8b1449ac912c025a984750c5dbff157
+`[1:]))
+
+	assert.NoError(t, err)
+	assert.Nil(t, kubernetes)
+	assert.Equal(t, &model.Container{ID: "7e9139716d9e5d762d22f9f877b87d1be8b1449ac912c025a984750c5dbff157"}, container)
+}
+
+func TestCgroupContainerInfoNonContainer(t *testing.T) {
+	container, _, err := readCgroupContainerInfo(strings.NewReader(`
+12:devices:/user.slice
+11:hugetlb:/
+10:memory:/user.slice
+9:freezer:/
+8:perf_event:/
+7:blkio:/user.slice
+6:pids:/user.slice/user-1000.slice/session-2.scope
+5:rdma:/
+4:cpuset:/
+3:net_cls,net_prio:/
+2:cpu,cpuacct:/user.slice
+1:name=systemd:/user.slice/user-1000.slice/session-2.scope
+0::/user.slice/user-1000.slice/session-2.scope`[1:]))
+
+	assert.NoError(t, err)
+	assert.Nil(t, kubernetes)
+	assert.Nil(t, container)
+}
+
+func TestCgroupContainerInfoDockerSystemd(t *testing.T) {
+	container, kubernetes, err := readCgroupContainerInfo(strings.NewReader(`
+1:name=systemd:/system.slice/docker-cde7c2bab394630a42d73dc610b9c57415dced996106665d427f6d0566594411.scope
+`[1:]))
+
+	assert.NoError(t, err)
+	assert.Nil(t, kubernetes)
+	assert.Equal(t, &model.Container{ID: "cde7c2bab394630a42d73dc610b9c57415dced996106665d427f6d0566594411"}, container)
+}
+
+func TestCgroupContainerInfoNonHex(t *testing.T) {
+	// Imaginary future format. We use the last part of the path,
+	// trimming legacy prefix/suffix, and check the expected
+	// length and runes used.
+	container, kubernetes, err := readCgroupContainerInfo(strings.NewReader(`
+1:name=systemd:/docker/051e2ee0bce99116029a13df4a9e943137f19f957f38ac02d6bad96f9b700f76/not_hex
+`[1:]))
+
+	assert.NoError(t, err)
+	assert.Nil(t, kubernetes)
+	assert.Nil(t, container)
+}
+
+func TestCgroupContainerInfoKubernetes(t *testing.T) {
+	hostname, err := os.Hostname()
+	require.NoError(t, err)
+	container, kubernetes, err := readCgroupContainerInfo(strings.NewReader(`
+1:name=systemd:/kubepods/besteffort/pode9b90526-f47d-11e8-b2a5-080027b9f4fb/15aa6e53-b09a-40c7-8558-c6c31e36c88a`[1:]))
+
+	assert.NoError(t, err)
+	assert.Equal(t, &model.Container{ID: "15aa6e53-b09a-40c7-8558-c6c31e36c88a"}, container)
+	assert.Equal(t, &model.Kubernetes{
+		Pod: &model.KubernetesPod{
+			UID:  "e9b90526-f47d-11e8-b2a5-080027b9f4fb",
+			Name: hostname,
+		},
+	}, kubernetes)
+}
+
+func TestCgroupContainerInfoKubernetesSystemd(t *testing.T) {
+	hostname, err := os.Hostname()
+	require.NoError(t, err)
+	container, kubernetes, err := readCgroupContainerInfo(strings.NewReader(`
+1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope`[1:]))
+
+	assert.NoError(t, err)
+	assert.Equal(t, &model.Container{ID: "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63"}, container)
+	assert.Equal(t, &model.Kubernetes{
+		Pod: &model.KubernetesPod{
+			UID:  "90d81341_92de_11e7_8cf2_507b9d4141fa",
+			Name: hostname,
+		},
+	}, kubernetes)
+}

internal/apmhostutil/container_nonlinux.go

@@ -0,0 +1,19 @@
+// +build !linux
+
+package apmhostutil
+
+import (
+	"runtime"
+
+	"github.com/pkg/errors"
+
+	"go.elastic.co/apm/model"
+)
+
+func containerInfo() (*model.Container, error) {
+	return nil, errors.Errorf("container ID computation not implemented for %s", runtime.GOOS)
+}
+
+func kubernetesInfo() (*model.Kubernetes, error) {
+	return nil, errors.Errorf("kubernetes info gathering not implemented for %s", runtime.GOOS)
+}

internal/apmhostutil/container_test.go

@@ -15,8 +15,8 @@ import (
 
 func TestContainerID(t *testing.T) {
 	if runtime.GOOS != "linux" {
-		// Currently we only support ContainerID in Linux containers.
-		_, err := apmhostutil.ContainerID()
+		// Currently we only support Container in Linux containers.
+		_, err := apmhostutil.Container()
 		assert.Error(t, err)
 		return
 	}
@@ -35,13 +35,14 @@ func TestContainerID(t *testing.T) {
 		t.Skipf("not running inside docker")
 	}
 
-	id, err := apmhostutil.ContainerID()
+	container, err := apmhostutil.Container()
 	require.NoError(t, err)
-	assert.Len(t, id, 64)
+	require.NotNil(t, container)
+	assert.Len(t, container.ID, 64)
 
 	// Docker sets the container hostname to a prefix
 	// of the full container ID.
 	hostname, err := os.Hostname()
 	require.NoError(t, err)
-	assert.Equal(t, hostname, id[:len(hostname)])
+	assert.Equal(t, hostname, container.ID[:len(hostname)])
 }