Skip to content

Commit

Permalink
[Heartbeat] Setuid to regular user / lower capabilities when possible (
Browse files Browse the repository at this point in the history
…#27878)

partial fix for #27648 , this PR:

Detects if the user is running as root then:
Checks to see if an environment variable BEAT_SETUID_AS (set in our Docker.tmpl) is present
Attempts to Setuid , Setgid and Setgroups to that user / groups
Invokes setcap to drop all privileges except NET_RAW+ep
This PR also fixes the broken syscall filtering in heartbeat, some non-syscall strings were breaking that.

With the changes here elastic-agent can still run as root, but the subprocesses can lower their privileges ASAP. This should also make it possible for heartbeat to safely run ICMP pings and synthetics. Synthetics must run as non-root, but ICMP requires NET_RAW. This lets us be consistent in our docs with the recommendation that elastic-agent run as root.
  • Loading branch information
andrewvc committed Oct 13, 2021
1 parent 058c405 commit a78a980
Show file tree
Hide file tree
Showing 16 changed files with 1,095 additions and 103 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro

- Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
- Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
- Fix broken seccomp filtering and improve security via `setcap` and `setuid` when running as root on linux in containers. {pull}27878[27878]

*Journalbeat*

Expand Down
812 changes: 812 additions & 0 deletions NOTICE.txt

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions dev-tools/notice/overrides.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,5 @@
{"name": "github.com/munnerz/goautoneg", "licenceType": "BSD-3-Clause"}
{"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"}
{"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"}
{"name": "kernel.org/pub/linux/libs/security/libcap/cap", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
{"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
2 changes: 1 addition & 1 deletion dev-tools/packaging/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ shared:
dockerfile: 'Dockerfile.elastic-agent.tmpl'
docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
user: '{{ .BeatName }}'
linux_capabilities: ''
linux_capabilities: 'cap_net_raw+eip'
image_name: ''
files:
'elastic-agent.yml':
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
rm {{ $beatBinary }} && \
ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \
chmod 0755 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
{{- if .linux_capabilities }}
setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
{{- end }}
{{- range $i, $modulesd := .ModulesDirs }}
chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
{{- end }}
Expand All @@ -30,11 +27,20 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
{{- end }}
true

{{- if .linux_capabilities }}
# Since the beat is stored at the other end of a symlink we must follow the symlink first
# For security reasons setcap does not support symlinks. This is smart in the general case
# but in our specific case since we're building a trusted image from trusted binaries this is
# fine. Thus, we use readlink to follow the link and setcap on the actual binary
RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
{{- end }}

FROM {{ .from }}

# Contains the elastic agent image variant, an empty string for the standard variant
# or "complete" for the bigger one.
ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}}
ENV BEAT_SETUID_AS={{ .user }}

{{- if contains .from "ubi-minimal" }}
RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq && microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
Expand Down
12 changes: 9 additions & 3 deletions dev-tools/packaging/templates/docker/Dockerfile.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,19 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
find {{ $beatHome }} -type d -exec chmod 0755 {} \; && \
find {{ $beatHome }} -type f -exec chmod 0644 {} \; && \
chmod 0755 {{ $beatBinary }} && \
{{- if .linux_capabilities }}
setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
{{- end }}
{{- range $i, $modulesd := .ModulesDirs }}
chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
{{- end }}
chmod 0775 {{ $beatHome }}/data {{ $beatHome }}/logs

{{- if .linux_capabilities }}
# Since the beat is stored at the other end of a symlink we must follow the symlink first
# For security reasons setcap does not support symlinks. This is smart in the general case
# but in our specific case since we're building a trusted image from trusted binaries this is
# fine. Thus, we use readlink to follow the link and setcap on the actual binary
RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
{{- end }}

FROM {{ .from }}

{{- if contains .from "ubi-minimal" }}
Expand Down Expand Up @@ -127,6 +132,7 @@ USER {{ .user }}
{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }}
# Setup synthetics env vars
ENV ELASTIC_SYNTHETICS_CAPABLE=true
ENV BEAT_SETUID_AS={{ .user }}
ENV SUITES_DIR={{ $beatHome }}/suites
ENV NODE_VERSION=14.17.5
ENV PATH="$NODE_PATH/node/bin:$PATH"
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ require (
k8s.io/api v0.21.1
k8s.io/apimachinery v0.21.1
k8s.io/client-go v0.21.1
kernel.org/pub/linux/libs/security/libcap/cap v1.2.57
)

require (
Expand Down Expand Up @@ -283,6 +284,7 @@ require (
k8s.io/klog/v2 v2.8.0 // indirect
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1344,6 +1344,10 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 h1:2nmqI+aw7EQZuelYktkQHBE4jESD2tOR+lOJEnv/Apo=
kernel.org/pub/linux/libs/security/libcap/cap v1.2.57/go.mod h1:uI99C3r4SXvJeuqoEtx/eWt7UbmfqqZ80H8q+9t/A7I=
kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 h1:NOFATXSf5z/cMR3HIwQ3Xrd3nwnWl5xThmNr5U/F0pI=
kernel.org/pub/linux/libs/security/libcap/psx v1.2.57/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
Expand Down
4 changes: 4 additions & 0 deletions heartbeat/beater/heartbeat.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package beater
import (
"errors"
"fmt"
"syscall"
"time"

"github.com/elastic/beats/v7/heartbeat/config"
Expand Down Expand Up @@ -81,6 +82,9 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
func (bt *Heartbeat) Run(b *beat.Beat) error {
logp.Info("heartbeat is running! Hit CTRL-C to stop it.")

groups, _ := syscall.Getgroups()
logp.Info("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups)

stopStaticMonitors, err := bt.RunStaticMonitors(b)
if err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion heartbeat/scripts/mage/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func CustomizePackaging() {
pkgType := args.Types[0]
switch pkgType {
case devtools.Docker:
args.Spec.ExtraVar("linux_capabilities", "cap_net_raw=eip")
args.Spec.ExtraVar("linux_capabilities", "cap_net_raw+eip")
args.Spec.Files[monitorsDTarget] = monitorsD
case devtools.TarGz, devtools.Zip:
args.Spec.Files[monitorsDTarget] = monitorsD
Expand Down
237 changes: 237 additions & 0 deletions heartbeat/security.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// TODO: Support other architectures (e.g. arm)
//go:build linux
// +build linux

package main

import (
"fmt"
"os"
"os/user"
"runtime"
"strconv"
"syscall"

"kernel.org/pub/linux/libs/security/libcap/cap"

"github.com/elastic/beats/v7/libbeat/common/seccomp"
)

func init() {
// Here we set a bunch of linux specific security stuff.
// In the context of a container, where users frequently run as root, we follow BEAT_SETUID_AS to setuid/gid
// and add capabilities to make this actually run as a regular user. This also helps Node.js in synthetics, which
// does not want to run as root. It's also just generally more secure.
if localUserName := os.Getenv("BEAT_SETUID_AS"); localUserName != "" && syscall.Geteuid() == 0 {
err := changeUser(localUserName)
if err != nil {
panic(err)
}
}

// Attempt to set capabilities before we setup seccomp rules
// Note that we discard any errors because they are not actionable.
// The beat should use `getcap` at a later point to examine available capabilities
// rather than relying on errors from `setcap`
setCapabilities()

switch runtime.GOARCH {
case "amd64", "386":
err := setSeccompRules()
if err != nil {
panic(err)
}
}
}

func changeUser(localUserName string) error {
localUser, err := user.Lookup(localUserName)
if err != nil {
return fmt.Errorf("could not lookup '%s': %w", localUser, err)
}
localUserUid, err := strconv.Atoi(localUser.Uid)
if err != nil {
return fmt.Errorf("could not parse UID '%s' as int: %w", localUser.Uid, err)
}
localUserGid, err := strconv.Atoi(localUser.Gid)
if err != nil {
return fmt.Errorf("could not parse GID '%s' as int: %w", localUser.Uid, err)
}
// We include the root group because the docker image contains many directories (data,logs)
// that are owned by root:root with 0775 perms. The heartbeat user is in both groups
// in the container, but we need to repeat that here.
err = syscall.Setgroups([]int{localUserGid, 0})
if err != nil {
return fmt.Errorf("could not set groups: %w", err)
}

// Set the main group as localUserUid so new files created are owned by the user's group
err = syscall.Setgid(localUserGid)
if err != nil {
return fmt.Errorf("could not set gid to %d: %w", localUserGid, err)
}

// Note this is not the regular SetUID! Look at the 'cap' package docs for it, it preserves
// capabilities post-SetUID, which we use to lock things down immediately
err = cap.SetUID(localUserUid)
if err != nil {
return fmt.Errorf("could not setuid to %d: %w", localUserUid, err)
}

// This may not be necessary, but is good hygeine, we do some shelling out to node/npm etc.
// and $HOME should reflect the user's preferences
return os.Setenv("HOME", localUser.HomeDir)
}

func setCapabilities() error {
// Start with an empty capability set
newcaps := cap.NewSet()
// Both permitted and effective are required! Permitted makes the permmission
// possible to get, effective makes it 'active'
err := newcaps.SetFlag(cap.Permitted, true, cap.NET_RAW)
if err != nil {
return fmt.Errorf("error setting permitted setcap: %w", err)
}
err = newcaps.SetFlag(cap.Effective, true, cap.NET_RAW)
if err != nil {
return fmt.Errorf("error setting effective setcap: %w", err)
}

// We do not want these capabilities to be inherited by subprocesses
err = newcaps.SetFlag(cap.Inheritable, false, cap.NET_RAW)
if err != nil {
return fmt.Errorf("error setting inheritable setcap: %w", err)
}

// Apply the new capabilities to the current process (incl. all threads)
err = newcaps.SetProc()
if err != nil {
return fmt.Errorf("error setting new process capabilities via setcap: %w", err)
}

return nil
}

func setSeccompRules() error {
// We require a number of syscalls to run. This list was generated with
// mage build && env ELASTIC_SYNTHETICS_CAPABLE=true strace -f --output=syscalls ./heartbeat --path.config sample-synthetics-config/ -e
// then grepping for 'EPERM' in the 'syscalls' file.
syscalls := []string{
"access",
"arch_prctl",
"bind",
"brk",
"capget",
"chdir",
"chmod",
"chown",
"clone",
"close",
"connect",
"creat",
"dup2",
"epoll_ctl",
"epoll_pwait",
"eventfd2",
"execve",
"exit",
"faccessat",
"fadvise64",
"fallocate",
"fcntl",
"flock",
"fstat",
"fsync",
"futex",
"capget",
"getcwd",
"getdents64",
"getegid",
"geteuid",
"getgid",
"getpeername",
"getpgrp",
"getpid",
"getppid",
"getpriority",
"getrandom",
"getresuid",
"getresgid",
"getrusage",
"getsockname",
"gettid",
"getuid",
"ioctl",
"inotify_init",
"lchown",
"link",
"lseek",
"madvise",
"memfd_create",
"mkdir",
"mkdirat",
"mlock",
"mmap",
"mprotect",
"munmap",
"nanosleep",
"name_to_handle_at",
"newfstatat",
"openat",
"pipe",
"pipe2",
"poll",
"prctl",
"pread64",
"prlimit64",
"pwrite64",
"read",
"readlink",
"readlinkat",
"recvfrom",
"rename",
"rmdir",
"rt_sigaction",
"rt_sigprocmask",
"rt_sigreturn",
"sched_getaffinity",
"sched_getparam",
"sched_getscheduler",
"select",
"sendto",
"set_robust_list",
"set_tid_address",
"setpriority",
"setsid",
"sigaltstack",
"socket",
"socketpair",
"stat",
"statx",
"symlink",
"umask",
"uname",
"unlink",
"utimensat",
"write",
}

return seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...)
}
2 changes: 1 addition & 1 deletion packetbeat/scripts/mage/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func CustomizePackaging() {
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml)
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml)
case devtools.Docker:
args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip")
args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin+eip")
default:
panic(errors.Errorf("unhandled package type: %v", pkgType))
}
Expand Down
Loading

0 comments on commit a78a980

Please sign in to comment.