-
Notifications
You must be signed in to change notification settings - Fork 386
/
git.go
486 lines (459 loc) · 16.2 KB
/
git.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
package buildcontext
import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"slices"
"strings"
"github.com/earthly/earthly/analytics"
"github.com/earthly/earthly/cleanup"
"github.com/earthly/earthly/conslogging"
"github.com/earthly/earthly/domain"
"github.com/earthly/earthly/features"
"github.com/earthly/earthly/util/gitutil"
"github.com/earthly/earthly/util/llbutil"
"github.com/earthly/earthly/util/llbutil/llbfactory"
"github.com/earthly/earthly/util/llbutil/pllb"
"github.com/earthly/earthly/util/platutil"
"github.com/earthly/earthly/util/stringutil"
"github.com/earthly/earthly/util/syncutil/synccache"
"github.com/earthly/earthly/util/vertexmeta"
buildkitgitutil "github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
const (
defaultGitImage = "alpine/git:v2.43.0"
)
type gitResolver struct {
cleanCollection *cleanup.Collection
gitBranchOverride string
lfsInclude string
logLevel buildkitgitutil.GitLogLevel
gitImage string
projectCache *synccache.SyncCache // "gitURL#gitRef" -> *resolvedGitProject
buildFileCache *synccache.SyncCache // project ref -> local path
gitLookup *GitLookup
console conslogging.ConsoleLogger
}
type resolvedGitProject struct {
// hash is the git hash.
hash string
// shortHash is the short git hash.
shortHash string
// branches is the git branches.
branches []string
// tags is the git tags.
tags []string
// committerTs is the git committer timestamp.
committerTs string
// authorTs is the git author timestamp.
authorTs string
authorEmail string
authorName string
coAuthors []string
// refs is the git refs
refs []string
// state is the state holding the git files.
state pllb.State
earthfilePaths []string
}
func (gr *gitResolver) expandWildcard(ctx context.Context, gwClient gwclient.Client, platr *platutil.Resolver, target domain.Target, pattern string) ([]string, error) {
if !target.IsRemote() {
return nil, errors.Errorf("unexpected local reference %s", target.String())
}
rgp, _, subDir, err := gr.resolveGitProject(ctx, gwClient, platr, target)
if err != nil {
return nil, errors.Wrapf(err, "failed resolving git project [platform: %s/%s]",
platr.LLBNative().OS, platr.LLBNative().Architecture)
}
fullPattern := filepath.Join(subDir, pattern)
if !strings.HasPrefix(fullPattern, ".") {
fullPattern = "./" + fullPattern
}
var matches []string
for _, path := range rgp.earthfilePaths {
path = strings.TrimSuffix(path, "/Earthfile")
ok, err := filepath.Match(fullPattern, path)
if err != nil {
return nil, err
}
if ok {
// Ensure we return paths that are relative to the sub-directory.
path = strings.TrimPrefix(path, fmt.Sprintf("./%s/", subDir))
matches = append(matches, path)
}
}
return matches, nil
}
func (gr *gitResolver) resolveEarthProject(ctx context.Context, gwClient gwclient.Client, platr *platutil.Resolver, ref domain.Reference, featureFlagOverrides string) (*Data, error) {
if !ref.IsRemote() {
return nil, errors.Errorf("unexpected local reference %s", ref.String())
}
rgp, gitURL, subDir, err := gr.resolveGitProject(ctx, gwClient, platr, ref)
if err != nil {
return nil, errors.Wrapf(err, "failed resolving git project [platform: %s/%s]",
platr.LLBNative().OS, platr.LLBNative().Architecture)
}
var buildContextFactory llbfactory.Factory
if _, isTarget := ref.(domain.Target); isTarget {
// Restrict the resulting build context to the right subdir.
if subDir == "." {
// Optimization.
buildContextFactory = llbfactory.PreconstructedState(rgp.state)
} else {
vm := &vertexmeta.VertexMeta{
TargetName: ref.String(),
Internal: true,
}
copyState, err := llbutil.CopyOp(ctx,
rgp.state, []string{subDir}, platr.Scratch(), "./", false, false, false, "root:root", nil, false, false, false,
llb.WithCustomNamef("%sCOPY git context %s", vm.ToVertexPrefix(), ref.String()))
if err != nil {
return nil, errors.Wrap(err, "copyOp failed in resolveEarthProject")
}
buildContextFactory = llbfactory.PreconstructedState(copyState)
}
}
// Else not needed: Commands don't come with a build context.
key := ref.ProjectCanonical()
isDockerfile := strings.HasPrefix(ref.GetName(), DockerfileMetaTarget)
if isDockerfile {
// Different key for dockerfiles to include the dockerfile name itself.
key = ref.StringCanonical()
}
localBuildFileValue, err := gr.buildFileCache.Do(ctx, key, func(ctx context.Context, _ interface{}) (interface{}, error) {
earthfileTmpDir, err := os.MkdirTemp(os.TempDir(), "earthly-git")
if err != nil {
return nil, errors.Wrap(err, "create temp dir for Earthfile")
}
gr.cleanCollection.Add(func() error {
return os.RemoveAll(earthfileTmpDir)
})
gitState, err := llbutil.StateToRef(
ctx, gwClient, rgp.state, false,
platr.SubResolver(platutil.NativePlatform), nil)
if err != nil {
return nil, errors.Wrap(err, "state to ref git meta")
}
bf, err := detectBuildFileInRef(ctx, ref, gitState, subDir)
if err != nil {
return nil, err
}
bfBytes, err := gitState.ReadFile(ctx, gwclient.ReadRequest{
Filename: bf,
})
if err != nil {
return nil, errors.Wrap(err, "read build file")
}
localBuildFilePath := filepath.Join(earthfileTmpDir, path.Base(bf))
err = os.WriteFile(localBuildFilePath, bfBytes, 0700)
if err != nil {
return nil, errors.Wrapf(err, "write build file to tmp dir at %s", localBuildFilePath)
}
var ftrs *features.Features
if isDockerfile {
ftrs = new(features.Features)
} else {
ftrs, err = parseFeatures(localBuildFilePath, featureFlagOverrides, ref.ProjectCanonical(), gr.console)
if err != nil {
return nil, err
}
}
return &buildFile{
path: localBuildFilePath,
ftrs: ftrs,
}, nil
})
if err != nil {
return nil, err
}
localBuildFile := localBuildFileValue.(*buildFile)
// TODO: Apply excludes / .earthignore.
return &Data{
BuildFilePath: localBuildFile.path,
BuildContextFactory: buildContextFactory,
GitMetadata: &gitutil.GitMetadata{
BaseDir: "",
RelDir: subDir,
RemoteURL: gitURL,
Hash: rgp.hash,
ShortHash: rgp.shortHash,
BranchOverrideTagArg: gr.gitBranchOverride != "",
Branch: rgp.branches,
Tags: rgp.tags,
CommitterTimestamp: rgp.committerTs,
AuthorTimestamp: rgp.authorTs,
AuthorEmail: rgp.authorEmail,
AuthorName: rgp.authorName,
CoAuthors: rgp.coAuthors,
Refs: rgp.refs,
},
Features: localBuildFile.ftrs,
}, nil
}
func (gr *gitResolver) resolveGitProject(ctx context.Context, gwClient gwclient.Client, platr *platutil.Resolver, ref domain.Reference) (rgp *resolvedGitProject, gitURL string, subDir string, finalErr error) {
gitRef := ref.GetTag()
var err error
var keyScans []string
var sshCommand string
gitURL, subDir, keyScans, sshCommand, err = gr.gitLookup.GetCloneURL(ref.GetGitURL())
if err != nil {
return nil, "", "", errors.Wrap(err, "failed to get url for cloning")
}
analytics.Count("gitResolver.resolveEarthProject", "")
// Check the cache first.
scrubbedGITURL := stringutil.ScrubCredentials(gitURL)
cacheKey := fmt.Sprintf("%s#%s", scrubbedGITURL, gitRef)
rgpValue, err := gr.projectCache.Do(ctx, cacheKey, func(ctx context.Context, k interface{}) (interface{}, error) {
// Copy all Earthfile, build.earth and Dockerfile files.
vm := &vertexmeta.VertexMeta{
TargetName: cacheKey,
Internal: true,
}
gitOpts := []llb.GitOption{
llb.WithCustomNamef("%sGIT CLONE %s", vm.ToVertexPrefix(), scrubbedGITURL),
llb.KeepGitDir(),
llb.LogLevel(gr.logLevel),
}
if len(keyScans) > 0 {
gitOpts = append(gitOpts, llb.KnownSSHHosts(strings.Join(keyScans, "\n")))
}
if gr.lfsInclude != "" {
// TODO this should eventually be inferred by the contents of a COPY command, which means the call to resolveGitProject will need to be lazy-evaluated
// However this makes it really difficult for an Earthfile which first has an ARG EARTHLY_GIT_HASH, then a RUN, then a COPY
gitOpts = append(gitOpts, llb.LFSInclude(gr.lfsInclude))
}
if sshCommand != "" {
gitOpts = append(gitOpts, llb.SSHCommand(sshCommand))
}
gitState := llb.Git(gitURL, gitRef, gitOpts...)
gitImage := gr.gitImage
if gitImage == "" {
gitImage = defaultGitImage
}
opImg := pllb.Image(
gitImage, llb.MarkImageInternal, llb.ResolveModePreferLocal,
llb.Platform(platr.LLBNative()))
// Get git hash.
gitHashOpts := []llb.RunOption{
llb.Args([]string{
"/bin/sh", "-c",
"git rev-parse HEAD >/dest/git-hash ; " +
"uname -m >/dest/uname-m ;" +
"git rev-parse --short=8 HEAD >/dest/git-short-hash ; " +
"git rev-parse --abbrev-ref HEAD >/dest/git-branch || touch /dest/git-branch ; " +
"ls .git/refs/heads/ | head -n 1 >/dest/git-default-branch || touch /dest/git-default-branch ; " +
"git describe --exact-match --tags >/dest/git-tags || touch /dest/git-tags ; " +
"git log -1 --format=%ct >/dest/git-committer-ts || touch /dest/git-committer-ts ; " +
"git log -1 --format=%at >/dest/git-author-ts || touch /dest/git-author-ts ; " +
"git log -1 --format=%ae >/dest/git-author-email || touch /dest/git-author-email ; " +
"git log -1 --format=%an >/dest/git-author-name || touch /dest/git-author-name ; " +
"git log -1 --format=%b >/dest/git-body || touch /dest/git-body ; " +
"git for-each-ref --points-at HEAD --format '%(refname:lstrip=-1)' >/dest/git-refs || touch /dest/git-refs ; " +
"find -type f -name Earthfile > /dest/Earthfile-paths || touch /dest/Earthfile-paths ; " +
"",
}),
llb.Dir("/git-src"),
llb.ReadonlyRootFS(),
llb.AddMount("/git-src", gitState, llb.Readonly),
llb.WithCustomNamef("%sGET GIT META %s", vm.ToVertexPrefix(), ref.ProjectCanonical()),
}
gitHashOp := opImg.Run(gitHashOpts...)
gitMetaState := gitHashOp.AddMount("/dest", platr.Scratch())
noCache := false // TODO figure out if we want to propagate --no-cache here
gitMetaRef, err := llbutil.StateToRef(
ctx, gwClient, gitMetaState, noCache,
platr.SubResolver(platutil.NativePlatform), nil)
if err != nil {
return nil, errors.Wrap(err, "state to ref git meta")
}
unameM, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "uname-m",
})
if err != nil {
return nil, errors.Wrap(err, "read uname-m")
}
var imgArch string
switch string(unameM) {
case "aarch64", "arm64":
imgArch = "arm64"
case "x86", "x86_64", "x64", "amd64":
imgArch = "amd64"
}
if imgArch != "" && imgArch != platr.LLBNative().Architecture {
gr.console.Warnf("git image [%s] has architecture [%s] which does not match host architecture [%s]",
gitImage, string(unameM), platr.LLBNative().Architecture)
}
gitHashBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-hash",
})
if err != nil {
return nil, errors.Wrap(err, "read git-hash")
}
gitShortHashBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-short-hash",
})
if err != nil {
return nil, errors.Wrap(err, "read git-short-hash")
}
gitBranch, err := gr.readGitBranch(ctx, gitMetaRef)
if err != nil {
return nil, errors.Wrap(err, "read git-branch")
}
gitDefaultBranchBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-default-branch",
})
if err != nil {
return nil, errors.Wrap(err, "read git-default-branch")
}
gitTagsBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-tags",
})
if err != nil {
return nil, errors.Wrap(err, "read git-tags")
}
gitCommitterTsBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-committer-ts",
})
if err != nil {
return nil, errors.Wrap(err, "read git-committer-ts")
}
gitAuthorTsBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-author-ts",
})
if err != nil {
return nil, errors.Wrap(err, "read git-author-ts")
}
gitAuthorEmailBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-author-email",
})
if err != nil {
return nil, errors.Wrap(err, "read git-author-email")
}
gitAuthorNameBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-author-name",
})
if err != nil {
return nil, errors.Wrap(err, "read git-author-name")
}
gitBodyBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-body",
})
if err != nil {
return nil, errors.Wrap(err, "read git-body")
}
gitRefsBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-refs",
})
if err != nil {
return nil, errors.Wrap(err, "read git-refs")
}
earthfilePathsRaw, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "Earthfile-paths",
})
if err != nil {
return nil, errors.Wrap(err, "read Earthfile-paths")
}
gitHash := strings.SplitN(string(gitHashBytes), "\n", 2)[0]
gitShortHash := strings.SplitN(string(gitShortHashBytes), "\n", 2)[0]
gitBranches := strings.SplitN(gitBranch, "\n", 2)
gitAuthorEmail := strings.SplitN(string(gitAuthorEmailBytes), "\n", 2)[0]
gitAuthorName := strings.SplitN(string(gitAuthorNameBytes), "\n", 2)[0]
gitCoAuthors := gitutil.ParseCoAuthorsFromBody(string(gitBodyBytes))
var gitBranches2 []string
for _, gitBranch := range gitBranches {
if gitBranch != "" && gitBranch != "HEAD" {
gitBranches2 = append(gitBranches2, gitBranch)
}
}
if len(gitBranches2) == 0 {
// fallback case for when git rev-parse --abbrev-ref fails
if gitRef != "" {
// use the reference name (if given); but only if it is not the git sha
if !strings.HasPrefix(gitRef, gitShortHash) {
gitBranches2 = []string{gitRef}
}
} else {
gitBranches2 = []string{strings.SplitN(string(gitDefaultBranchBytes), "\n", 2)[0]}
}
}
gitTags := strings.SplitN(string(gitTagsBytes), "\n", 2)
var gitTags2 []string
for _, gitTag := range gitTags {
if gitTag != "" && gitTag != "HEAD" {
gitTags2 = append(gitTags2, gitTag)
}
}
gitCommiterTs := strings.SplitN(string(gitCommitterTsBytes), "\n", 2)[0]
gitAuthorTs := strings.SplitN(string(gitAuthorTsBytes), "\n", 2)[0]
gitRefs := strings.Split(string(gitRefsBytes), "\n")
var gitRefs2 []string
for _, gitRef := range gitRefs {
gitRef = strings.Trim(gitRef, "'\"")
if gitRef != "" && gitRef != "HEAD" && !slices.Contains(gitRefs2, gitRef) {
gitRefs2 = append(gitRefs2, gitRef)
}
}
gitOpts = []llb.GitOption{
llb.WithCustomNamef("[context %s] git context %s", scrubbedGITURL, ref.StringCanonical()),
llb.KeepGitDir(),
}
if len(keyScans) > 0 {
gitOpts = append(gitOpts, llb.KnownSSHHosts(strings.Join(keyScans, "\n")))
}
if gr.lfsInclude != "" {
gitOpts = append(gitOpts, llb.LFSInclude(gr.lfsInclude))
}
rgp := &resolvedGitProject{
hash: gitHash,
shortHash: gitShortHash,
branches: gitBranches2,
tags: gitTags2,
committerTs: gitCommiterTs,
authorTs: gitAuthorTs,
authorEmail: gitAuthorEmail,
authorName: gitAuthorName,
coAuthors: gitCoAuthors,
refs: gitRefs2,
earthfilePaths: strings.Split(strings.TrimSpace(string(earthfilePathsRaw)), "\n"),
state: pllb.Git(
gitURL,
gitHash,
gitOpts...,
),
}
go func() {
// Add cache entries for the branch and for the tag (if any).
if len(gitBranches2) > 0 {
cacheKey3 := fmt.Sprintf("%s#%s", scrubbedGITURL, gitBranches2[0])
_ = gr.projectCache.Add(ctx, cacheKey3, rgp, nil)
}
if len(gitTags2) > 0 {
cacheKey4 := fmt.Sprintf("%s#%s", scrubbedGITURL, gitTags2[0])
_ = gr.projectCache.Add(ctx, cacheKey4, rgp, nil)
}
}()
return rgp, nil
})
if err != nil {
return nil, "", "", err
}
rgp = rgpValue.(*resolvedGitProject)
return rgp, gitURL, subDir, nil
}
func (gr *gitResolver) readGitBranch(ctx context.Context, gitMetaRef gwclient.Reference) (string, error) {
if gr.gitBranchOverride != "" {
return gr.gitBranchOverride, nil
}
gitBranchBytes, err := gitMetaRef.ReadFile(ctx, gwclient.ReadRequest{
Filename: "git-branch",
})
if err != nil {
return "", err
}
return string(gitBranchBytes), nil
}