forked from hashicorp/nomad
/
executor_universal_linux.go
193 lines (166 loc) · 5.69 KB
/
executor_universal_linux.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
package executor
import (
"fmt"
"os"
"os/user"
"strconv"
"syscall"
"github.com/containernetworking/plugins/pkg/ns"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/plugins/drivers"
"github.com/opencontainers/runc/libcontainer/cgroups"
cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
lconfigs "github.com/opencontainers/runc/libcontainer/configs"
)
// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
u, err := user.Lookup(userid)
if err != nil {
return fmt.Errorf("Failed to identify user %v: %v", userid, err)
}
// Get the groups the user is a part of
gidStrings, err := u.GroupIds()
if err != nil {
return fmt.Errorf("Unable to lookup user's group membership: %v", err)
}
gids := make([]uint32, len(gidStrings))
for _, gidString := range gidStrings {
u, err := strconv.Atoi(gidString)
if err != nil {
return fmt.Errorf("Unable to convert user's group to int %s: %v", gidString, err)
}
gids = append(gids, uint32(u))
}
// Convert the uid and gid
uid, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert userid to uint32: %s", err)
}
gid, err := strconv.ParseUint(u.Gid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
}
// Set the command to run as that user and group.
if e.childCmd.SysProcAttr == nil {
e.childCmd.SysProcAttr = &syscall.SysProcAttr{}
}
if e.childCmd.SysProcAttr.Credential == nil {
e.childCmd.SysProcAttr.Credential = &syscall.Credential{}
}
e.childCmd.SysProcAttr.Credential.Uid = uint32(uid)
e.childCmd.SysProcAttr.Credential.Gid = uint32(gid)
e.childCmd.SysProcAttr.Credential.Groups = gids
e.logger.Debug("setting process user", "user", uid, "group", gid, "additional_groups", gids)
return nil
}
// configureResourceContainer configured the cgroups to be used to track pids
// created by the executor
func (e *UniversalExecutor) configureResourceContainer(pid int) error {
cfg := &lconfigs.Config{
Cgroups: &lconfigs.Cgroup{
Resources: &lconfigs.Resources{
AllowAllDevices: helper.BoolToPtr(true),
},
},
}
err := configureBasicCgroups(cfg)
if err != nil {
// Log this error to help diagnose cases where nomad is run with too few
// permissions, but don't return an error. There is no separate check for
// cgroup creation permissions, so this may be the happy path.
e.logger.Warn("failed to create cgroup",
"docs", "https://www.nomadproject.io/docs/drivers/raw_exec.html#no_cgroups",
"error", err)
return nil
}
e.resConCtx.groups = cfg.Cgroups
return cgroups.EnterPid(cfg.Cgroups.Paths, pid)
}
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
if e.resConCtx.isEmpty() {
return getAllPidsByScanning()
} else {
return e.resConCtx.getAllPidsByCgroup()
}
}
// DestroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host. This function is idempotent.
func DestroyCgroup(groups *lconfigs.Cgroup, executorPid int) error {
mErrs := new(multierror.Error)
if groups == nil {
return fmt.Errorf("Can't destroy: cgroup configuration empty")
}
// Move the executor into the global cgroup so that the task specific
// cgroup can be destroyed.
path, err := cgroups.GetInitCgroupPath("freezer")
if err != nil {
return err
}
if err := cgroups.EnterPid(map[string]string{"freezer": path}, executorPid); err != nil {
return err
}
// Freeze the Cgroup so that it can not continue to fork/exec.
groups.Resources.Freezer = lconfigs.Frozen
freezer := cgroupFs.FreezerGroup{}
if err := freezer.Set(groups.Paths[freezer.Name()], groups); err != nil {
return err
}
var procs []*os.Process
pids, err := cgroups.GetAllPids(groups.Paths[freezer.Name()])
if err != nil {
multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))
// Unfreeze the cgroup.
groups.Resources.Freezer = lconfigs.Thawed
freezer := cgroupFs.FreezerGroup{}
if err := freezer.Set(groups.Paths[freezer.Name()], groups); err != nil {
multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
return mErrs.ErrorOrNil()
}
}
// Kill the processes in the cgroup
for _, pid := range pids {
proc, err := os.FindProcess(pid)
if err != nil {
multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
continue
}
procs = append(procs, proc)
if e := proc.Kill(); e != nil {
multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
}
}
// Unfreeze the cgroug so we can wait.
groups.Resources.Freezer = lconfigs.Thawed
if err := freezer.Set(groups.Paths[freezer.Name()], groups); err != nil {
multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
return mErrs.ErrorOrNil()
}
// Wait on the killed processes to ensure they are cleaned up.
for _, proc := range procs {
// Don't capture the error because we expect this to fail for
// processes we didn't fork.
proc.Wait()
}
// Remove the cgroup.
if err := cgroups.RemovePaths(groups.Paths); err != nil {
multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
}
return mErrs.ErrorOrNil()
}
// withNetworkIsolation calls the passed function the network namespace `spec`
func withNetworkIsolation(f func() error, spec *drivers.NetworkIsolationSpec) error {
if spec != nil && spec.Path != "" {
// Get a handle to the target network namespace
netns, err := ns.GetNS(spec.Path)
if err != nil {
return err
}
// Start the container in the network namespace
return netns.Do(func(ns.NetNS) error {
return f()
})
}
return f()
}