-
Notifications
You must be signed in to change notification settings - Fork 226
/
fetch.go
257 lines (228 loc) · 8.53 KB
/
fetch.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/otiai10/copy"
"github.com/GoogleContainerTools/kpt/internal/errors"
"github.com/GoogleContainerTools/kpt/internal/gitutil"
"github.com/GoogleContainerTools/kpt/internal/pkg"
"github.com/GoogleContainerTools/kpt/internal/printer"
"github.com/GoogleContainerTools/kpt/internal/types"
"github.com/GoogleContainerTools/kpt/internal/util/git"
"github.com/GoogleContainerTools/kpt/internal/util/pkgutil"
kptfilev1 "github.com/GoogleContainerTools/kpt/pkg/api/kptfile/v1"
"github.com/GoogleContainerTools/kpt/pkg/kptfile/kptfileutil"
)
// Command takes the upstream information in the Kptfile at the path for the
// provided package, and fetches the package referenced if it isn't already
// there.
type Command struct {
Pkg *pkg.Pkg
}
// Run runs the Command.
func (c Command) Run(ctx context.Context) error {
const op errors.Op = "fetch.Run"
kf, err := c.Pkg.Kptfile()
if err != nil {
return errors.E(op, c.Pkg.UniquePath, fmt.Errorf("no Kptfile found"))
}
if err := c.validate(kf); err != nil {
return errors.E(op, c.Pkg.UniquePath, err)
}
g := kf.Upstream.Git
repoSpec := &git.RepoSpec{
OrgRepo: g.Repo,
Path: g.Directory,
Ref: g.Ref,
}
err = cloneAndCopy(ctx, repoSpec, c.Pkg.UniquePath.String())
if err != nil {
return errors.E(op, c.Pkg.UniquePath, err)
}
return nil
}
// validate makes sure the Kptfile has the necessary information to fetch
// the package.
func (c Command) validate(kf *kptfilev1.KptFile) error {
const op errors.Op = "validate"
if kf.Upstream == nil {
return errors.E(op, errors.MissingParam, fmt.Errorf("kptfile doesn't contain upstream information"))
}
if kf.Upstream.Git == nil {
return errors.E(op, errors.MissingParam, fmt.Errorf("kptfile upstream doesn't have git information"))
}
g := kf.Upstream.Git
if len(g.Repo) == 0 {
return errors.E(op, errors.MissingParam, fmt.Errorf("must specify repo"))
}
if len(g.Ref) == 0 {
return errors.E(op, errors.MissingParam, fmt.Errorf("must specify ref"))
}
if len(g.Directory) == 0 {
return errors.E(op, errors.MissingParam, fmt.Errorf("must specify directory"))
}
return nil
}
// cloneAndCopy fetches the provided repo and copies the content into the
// directory specified by dest. The provided name is set as `metadata.name`
// of the Kptfile of the package.
func cloneAndCopy(ctx context.Context, r *git.RepoSpec, dest string) error {
const op errors.Op = "fetch.cloneAndCopy"
pr := printer.FromContextOrDie(ctx)
err := ClonerUsingGitExec(ctx, r)
if err != nil {
return errors.E(op, errors.Git, types.UniquePath(dest), err)
}
defer os.RemoveAll(r.Dir)
sourcePath := filepath.Join(r.Dir, r.Path)
pr.Printf("Adding package %q.\n", strings.TrimPrefix(r.Path, "/"))
if err := pkgutil.CopyPackage(sourcePath, dest, true, pkg.All); err != nil {
return errors.E(op, types.UniquePath(dest), err)
}
if err := kptfileutil.UpdateKptfileWithoutOrigin(dest, sourcePath, false); err != nil {
return errors.E(op, types.UniquePath(dest), err)
}
if err := kptfileutil.UpdateUpstreamLockFromGit(dest, r); err != nil {
return errors.E(op, errors.Git, types.UniquePath(dest), err)
}
return nil
}
// ClonerUsingGitExec uses a local git install, as opposed
// to say, some remote API, to obtain a local clone of
// a remote repo. It looks for tags with the directory as a prefix to allow
// for versioning multiple kpt packages in a single repo independently. It
// relies on the private clonerUsingGitExec function to try fetching different
// refs.
func ClonerUsingGitExec(ctx context.Context, repoSpec *git.RepoSpec) error {
const op errors.Op = "fetch.ClonerUsingGitExec"
// Create a local representation of the upstream repo. This will initialize
// the cache for the specified repo uri if it isn't already there. It also
// fetches and caches all tag and branch refs from the upstream repo.
upstreamRepo, err := gitutil.NewGitUpstreamRepo(ctx, repoSpec.CloneSpec())
if err != nil {
return errors.E(op, errors.Git, errors.Repo(repoSpec.CloneSpec()), err)
}
// Check if we have a ref in the upstream that matches the package-specific
// reference. If we do, we use that reference.
ps := strings.Split(repoSpec.Path, "/")
for len(ps) != 0 {
p := path.Join(ps...)
packageRef := path.Join(strings.TrimLeft(p, "/"), repoSpec.Ref)
if _, found := upstreamRepo.ResolveTag(packageRef); found {
repoSpec.Ref = packageRef
break
}
ps = ps[:len(ps)-1]
}
// Pull the required ref into the repo git cache.
dir, err := upstreamRepo.GetRepo(ctx, []string{repoSpec.Ref})
if err != nil {
return errors.E(op, errors.Git, errors.Repo(repoSpec.CloneSpec()), err)
}
gitRunner, err := gitutil.NewLocalGitRunner(dir)
if err != nil {
return errors.E(op, errors.Git, errors.Repo(repoSpec.CloneSpec()), err)
}
// Find the commit SHA for the ref that was just fetched. We need the SHA
// rather than the ref to be able to do a hard reset of the cache repo.
commit, found := upstreamRepo.ResolveRef(repoSpec.Ref)
if !found {
commit = repoSpec.Ref
}
// Reset the local repo to the commit we need. Doing a hard reset instead of
// a checkout means we don't create any local branches so we don't need to
// worry about fast-forwarding them with changes from upstream. It also makes
// sure that any changes in the local worktree are cleaned out.
_, err = gitRunner.Run(ctx, "reset", "--hard", commit)
if err != nil {
gitutil.AmendGitExecError(err, func(e *gitutil.GitExecError) {
e.Repo = repoSpec.CloneSpec()
e.Ref = commit
})
return errors.E(op, errors.Git, errors.Repo(repoSpec.CloneSpec()), err)
}
// We need to create a temp directory where we can copy the content of the repo.
// During update, we need to checkout multiple versions of the same repo, so
// we can't do merges directly from the cache.
repoSpec.Dir, err = ioutil.TempDir("", "kpt-get-")
if err != nil {
return errors.E(op, errors.Internal, fmt.Errorf("error creating temp directory: %w", err))
}
repoSpec.Commit = commit
pkgPath := filepath.Join(dir, repoSpec.Path)
// Verify that the requested path exists in the repo.
_, err = os.Stat(pkgPath)
if os.IsNotExist(err) {
return errors.E(op,
errors.Internal,
err,
fmt.Errorf("path %q does not exist in repo %q", repoSpec.Path, repoSpec.OrgRepo))
}
// Copy the content of the pkg into the temp directory.
// Note that we skip the content outside the package directory.
err = copyDir(ctx, pkgPath, repoSpec.AbsPath())
if err != nil {
return errors.E(op, errors.Internal, fmt.Errorf("error copying package: %w", err))
}
// Verify that if a Kptfile exists in the package, it contains the correct
// version of the Kptfile.
_, err = pkg.ReadKptfile(pkgPath)
if err != nil {
// A Kptfile isn't required, so it is fine if there is no Kptfile.
if errors.Is(err, os.ErrNotExist) {
return nil
}
// If the error is of type KptfileError, we replace it with a
// RemoteKptfileError. This allows us to provide information about the
// git source of the Kptfile instead of the path to some random
// temporary directory.
var kfError *pkg.KptfileError
if errors.As(err, &kfError) {
return &pkg.RemoteKptfileError{
RepoSpec: repoSpec,
Err: kfError.Err,
}
}
}
return nil
}
// copyDir copies a src directory to a dst directory.
// copyDir skips copying the .git directory from the src and ignores symlinks.
func copyDir(ctx context.Context, srcDir string, dstDir string) error {
pr := printer.FromContextOrDie(ctx)
opts := copy.Options{
Skip: func(src string) (bool, error) {
return strings.HasSuffix(src, ".git"), nil
},
OnSymlink: func(src string) copy.SymlinkAction {
// try to print relative path of symlink
// if we can, else absolute path which is not
// pretty because it contains path to temporary repo dir
displayPath, err := filepath.Rel(srcDir, src)
if err != nil {
displayPath = src
}
pr.Printf("[Warn] Ignoring symlink %q \n", displayPath)
return copy.Skip
},
}
return copy.Copy(srcDir, dstDir, opts)
}