/
plugin.go
413 lines (349 loc) · 12.8 KB
/
plugin.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
package job
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/buildkite/agent/v3/agent/plugin"
"github.com/buildkite/agent/v3/internal/job/hook"
"github.com/buildkite/agent/v3/internal/utils"
"github.com/buildkite/roko"
)
type pluginCheckout struct {
*plugin.Plugin
*plugin.Definition
CheckoutDir string
HooksDir string
}
func (e *Executor) hasPlugins() bool {
return e.ExecutorConfig.Plugins != ""
}
func (e *Executor) preparePlugins() error {
if !e.hasPlugins() {
return nil
}
e.shell.Headerf("Preparing plugins")
if e.Debug {
e.shell.Commentf("Plugin JSON is %s", e.Plugins)
}
// Check if we can run plugins (disabled via --no-plugins)
if !e.ExecutorConfig.PluginsEnabled {
if !e.ExecutorConfig.LocalHooksEnabled {
return fmt.Errorf("Plugins have been disabled on this agent with `--no-local-hooks`")
} else if !e.ExecutorConfig.CommandEval {
return fmt.Errorf("Plugins have been disabled on this agent with `--no-command-eval`")
} else {
return fmt.Errorf("Plugins have been disabled on this agent with `--no-plugins`")
}
}
var err error
e.plugins, err = plugin.CreateFromJSON(e.ExecutorConfig.Plugins)
if err != nil {
return fmt.Errorf("Failed to parse a plugin definition: %w", err)
}
if e.Debug {
e.shell.Commentf("Parsed %d plugins", len(e.plugins))
}
return nil
}
func (e *Executor) validatePluginCheckout(ctx context.Context, checkout *pluginCheckout) error {
if !e.ExecutorConfig.PluginValidation {
return nil
}
if checkout.Definition == nil {
if e.Debug {
e.shell.Commentf("Parsing plugin definition for %s from %s", checkout.Plugin.Name(), checkout.CheckoutDir)
}
// parse the plugin definition from the plugin checkout dir
var err error
checkout.Definition, err = plugin.LoadDefinitionFromDir(checkout.CheckoutDir)
if errors.Is(err, plugin.ErrDefinitionNotFound) {
e.shell.Warningf("Failed to find plugin definition for plugin %s", checkout.Plugin.Name())
return nil
} else if err != nil {
return err
}
}
val := &plugin.Validator{}
result := val.Validate(ctx, checkout.Definition, checkout.Plugin.Configuration)
if !result.Valid() {
e.shell.Headerf("Plugin validation failed for %q", checkout.Plugin.Name())
json, _ := json.Marshal(checkout.Plugin.Configuration)
e.shell.Commentf("Plugin configuration JSON is %s", json)
return result
}
e.shell.Commentf("Valid plugin configuration for %q", checkout.Plugin.Name())
return nil
}
// PluginPhase is where plugins that weren't filtered in the Environment phase are
// checked out and made available to later phases
func (e *Executor) PluginPhase(ctx context.Context) error {
if len(e.plugins) == 0 {
if e.Debug {
e.shell.Commentf("Skipping plugin phase")
}
return nil
}
checkouts := []*pluginCheckout{}
// Checkout and validate plugins that aren't vendored
for _, p := range e.plugins {
if p.Vendored {
if e.Debug {
e.shell.Commentf("Skipping vendored plugin %s", p.Name())
}
continue
}
checkout, err := e.checkoutPlugin(ctx, p)
if err != nil {
return fmt.Errorf("Failed to checkout plugin %s: %w", p.Name(), err)
}
err = e.validatePluginCheckout(ctx, checkout)
if err != nil {
return err
}
checkouts = append(checkouts, checkout)
}
// Store the checkouts for future use
e.pluginCheckouts = checkouts
// Now we can run plugin environment hooks too
return e.executePluginHook(ctx, "environment", checkouts)
}
// VendoredPluginPhase is where plugins that are included in the
// checked out code are added
func (e *Executor) VendoredPluginPhase(ctx context.Context) error {
if !e.hasPlugins() {
return nil
}
vendoredCheckouts := []*pluginCheckout{}
// Validate vendored plugins
for _, p := range e.plugins {
if !p.Vendored {
continue
}
checkoutPath, _ := e.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
pluginLocation, err := filepath.Abs(filepath.Join(checkoutPath, p.Location))
if err != nil {
return fmt.Errorf("Failed to resolve vendored plugin path for plugin %s: %w", p.Name(), err)
}
if !utils.FileExists(pluginLocation) {
return fmt.Errorf("Vendored plugin path %s doesn't exist", p.Location)
}
checkout := &pluginCheckout{
Plugin: p,
CheckoutDir: pluginLocation,
HooksDir: filepath.Join(pluginLocation, "hooks"),
}
// Also make sure that plugin is within this repository
// checkout and isn't elsewhere on the system.
if !strings.HasPrefix(pluginLocation, checkoutPath+string(os.PathSeparator)) {
return fmt.Errorf("Vendored plugin paths must be within the checked-out repository")
}
err = e.validatePluginCheckout(ctx, checkout)
if err != nil {
return err
}
vendoredCheckouts = append(vendoredCheckouts, checkout)
}
// Finally append our vendored checkouts to the rest for subsequent hooks
e.pluginCheckouts = append(e.pluginCheckouts, vendoredCheckouts...)
// Now we can run plugin environment hooks too
return e.executePluginHook(ctx, "environment", vendoredCheckouts)
}
// Hook types that we should only run one of, but a long-standing bug means that
// we allowed more than one to run (for plugins).
var strictSingleHookTypes = map[string]bool{
"command": true,
"checkout": true,
}
// Executes a named hook on plugins that have it
func (e *Executor) executePluginHook(ctx context.Context, name string, checkouts []*pluginCheckout) error {
// Command and checkout hooks are a little different, in that we only execute
// the first one we see. We run the first one, and output a warning for all
// the subsequent ones.
hookTypeSeen := make(map[string]bool)
for i, p := range checkouts {
hookPath, err := hook.Find(p.HooksDir, name)
if errors.Is(err, os.ErrNotExist) {
continue // this plugin does not implement this hook
}
if err != nil {
return err
}
if strictSingleHookTypes[name] && hookTypeSeen[name] {
if e.ExecutorConfig.StrictSingleHooks {
e.shell.Logger.Warningf("Ignoring additional %s hook (%s plugin, position %d)",
name, p.Plugin.Name(), i+1)
continue
} else {
e.shell.Logger.Warningf("The additional %s hook (%s plugin, position %d) "+
"will be ignored in a future version of the agent. To enforce "+
"single %s hooks now, pass the --strict-single-hooks flag, set "+
"the environment variable BUILDKITE_STRICT_SINGLE_HOOKS=true, "+
"or set strict-single-hooks=true in your agent configuration",
name, p.Plugin.Name(), i+1, name)
}
}
hookTypeSeen[name] = true
envMap, err := p.ConfigurationToEnvironment()
if dnerr := (&plugin.DeprecatedNameErrors{}); errors.As(err, &dnerr) {
e.shell.Logger.Headerf("Deprecated environment variables for plugin %s", p.Plugin.Name())
e.shell.Logger.Printf("%s", strings.Join([]string{
"The way that environment variables are derived from the plugin configuration is changing.",
"We'll export both the deprecated and the replacement names for now,",
"You may be able to avoid this by removing consecutive underscore, hyphen, or whitespace",
"characters in your plugin configuration.",
}, " "))
for _, err := range dnerr.Unwrap() {
e.shell.Logger.Printf("%s", err.Error())
}
} else if err != nil {
e.shell.Logger.Warningf("Error configuring plugin environment: %s", err)
}
if err := e.executeHook(ctx, HookConfig{
Scope: "plugin",
Name: name,
Path: hookPath,
Env: envMap,
PluginName: p.Plugin.Name(),
SpanAttributes: map[string]string{
"plugin.name": p.Plugin.Name(),
"plugin.version": p.Plugin.Version,
"plugin.location": p.Plugin.Location,
"plugin.is_vendored": strconv.FormatBool(p.Vendored),
},
}); err != nil {
return err
}
}
return nil
}
// If any plugin has a hook by this name
func (e *Executor) hasPluginHook(name string) bool {
for _, p := range e.pluginCheckouts {
if _, err := hook.Find(p.HooksDir, name); err == nil {
return true
}
}
return false
}
// Checkout a given plugin to the plugins directory and return that directory. Each agent worker
// will checkout the plugin to a different directory, so that they don't conflict with each other.
// Because the plugin directory is unique to the agent worker, we don't lock it. However, if
// multiple agent workers have access to the plugin directory, they need to have different names.
func (e *Executor) checkoutPlugin(ctx context.Context, p *plugin.Plugin) (*pluginCheckout, error) {
// Make sure we have a plugin path before trying to do anything
if e.PluginsPath == "" {
return nil, fmt.Errorf("Can't checkout plugin without a `plugins-path`")
}
id, err := p.Identifier()
if err != nil {
return nil, err
}
pluginParentDir := filepath.Join(e.PluginsPath, e.AgentName)
// Ensure the parent of the plugin directory exists, otherwise we can't move the temp git repo dir
// into it. The actual file permissions will be reduced by umask, and won't be 0o777 unless the
// user has manually changed the umask to 0o000
if err := os.MkdirAll(pluginParentDir, 0o777); err != nil {
return nil, err
}
// Create a path to the plugin
pluginDirectory := filepath.Join(pluginParentDir, id)
pluginGitDirectory := filepath.Join(pluginDirectory, ".git")
checkout := &pluginCheckout{
Plugin: p,
CheckoutDir: pluginDirectory,
HooksDir: filepath.Join(pluginDirectory, "hooks"),
}
// If there is already a clone, the user may want to ensure it's fresh (e.g., by setting
// BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH=true).
//
// Neither of the obvious options here is very nice. Either we git-fetch and git-checkout on
// existing repos, which is probably fast, but it's _surprisingly hard_ to write a really robust
// chain of Git commands that'll definitely get you a clean version of a given upstream branch
// ref (the branch might have been force-pushed, the checkout might have become dirty and
// unmergeable, etc.). Plus, then we're duplicating a bunch of fetch/checkout machinery and
// perhaps missing things (like `addRepositoryHostToSSHKnownHosts` which is called down below).
// Alternatively, we can DRY it up and simply `rm -rf` the plugin directory if it exists, but
// that means a potentially slow and unnecessary clone on every build step. Sigh. I think the
// tradeoff is favourable for just blowing away an existing clone if we want least-hassle
// guarantee that the user will get the latest version of their plugin branch/tag/whatever.
if e.ExecutorConfig.PluginsAlwaysCloneFresh && utils.FileExists(pluginDirectory) {
e.shell.Commentf("BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH is true; removing previous checkout of plugin %s", p.Label())
err = os.RemoveAll(pluginDirectory)
if err != nil {
e.shell.Errorf("Oh no, something went wrong removing %s", pluginDirectory)
return nil, err
}
}
if utils.FileExists(pluginGitDirectory) {
// It'd be nice to show the current commit of the plugin, so
// let's figure that out.
headCommit, err := gitRevParseInWorkingDirectory(ctx, e.shell, pluginDirectory, "--short=7", "HEAD")
if err != nil {
e.shell.Commentf("Plugin %q already checked out (can't `git rev-parse HEAD` plugin git directory)", p.Label())
} else {
e.shell.Commentf("Plugin %q already checked out (%s)", p.Label(), strings.TrimSpace(headCommit))
}
return checkout, nil
}
e.shell.Commentf("Plugin \"%s\" will be checked out to \"%s\"", p.Location, pluginDirectory)
repo, err := p.Repository()
if err != nil {
return nil, err
}
if e.SSHKeyscan {
addRepositoryHostToSSHKnownHosts(ctx, e.shell, repo)
}
// Make the directory
tempDir, err := os.MkdirTemp(e.PluginsPath, id)
if err != nil {
return nil, err
}
// Switch to the plugin directory
e.shell.Commentf("Switching to the temporary plugin directory")
previousWd := e.shell.Getwd()
if err := e.shell.Chdir(tempDir); err != nil {
return nil, err
}
// Switch back to the previous working directory
defer func() {
if err := e.shell.Chdir(previousWd); err != nil && e.Debug {
e.shell.Errorf("failed to switch back to previous working directory: %v", err)
}
}()
args := []string{"clone", "-v"}
if e.GitSubmodules {
// "--recursive" was added in Git 1.6.5, and is an alias to
// "--recurse-submodules" from Git 2.13.
args = append(args, "--recursive")
}
args = append(args, "--", repo, ".")
// Plugin clones shouldn't use custom GitCloneFlags
err = roko.NewRetrier(
roko.WithMaxAttempts(3),
roko.WithStrategy(roko.Constant(2*time.Second)),
).DoWithContext(ctx, func(r *roko.Retrier) error {
return e.shell.Run(ctx, "git", args...)
})
if err != nil {
return nil, err
}
// Switch to the version if we need to
if p.Version != "" {
e.shell.Commentf("Checking out `%s`", p.Version)
if err = e.shell.Run(ctx, "git", "checkout", "-f", p.Version); err != nil {
return nil, err
}
}
e.shell.Commentf("Moving temporary plugin directory to final location")
err = os.Rename(tempDir, pluginDirectory)
if err != nil {
return nil, err
}
return checkout, nil
}