-
Notifications
You must be signed in to change notification settings - Fork 784
/
clientfactory.go
323 lines (277 loc) · 10.7 KB
/
clientfactory.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
package metapipeline
import (
"fmt"
"io/ioutil"
"os"
"strings"
"time"
"github.com/jenkins-x/jx-logging/pkg/log"
"github.com/jenkins-x/jx/v2/pkg/apps"
"github.com/jenkins-x/jx/v2/pkg/client/clientset/versioned"
jxclient "github.com/jenkins-x/jx/v2/pkg/client/clientset/versioned"
"github.com/jenkins-x/jx/v2/pkg/config"
"github.com/jenkins-x/jx/v2/pkg/gits"
"github.com/jenkins-x/jx/v2/pkg/jxfactory"
"github.com/jenkins-x/jx/v2/pkg/kube"
"github.com/jenkins-x/jx/v2/pkg/tekton"
"github.com/jenkins-x/jx/v2/pkg/util"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
tektonclient "github.com/tektoncd/pipeline/pkg/client/clientset/versioned"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
kubeclient "k8s.io/client-go/kubernetes"
)
const (
retryDuration = time.Second * 30
defaultCheckoutDir = "source"
)
var (
logger = log.Logger().WithFields(logrus.Fields{"component": "meta-pipeline-client"})
)
// clientFactory implements the interface methods to create and apply the meta pipeline.
type clientFactory struct {
jxClient versioned.Interface
tektonClient tektonclient.Interface
kubeClient kubernetes.Interface
ns string
versionDir string
versionStreamURL string
versionStreamRef string
}
// NewMetaPipelineClient creates a new client for the creation and application of meta pipelines.
// The responsibility of the meta pipeline is to prepare the execution pipeline and to allow Apps to contribute
// the this execution pipeline.
func NewMetaPipelineClient() (Client, error) {
tektonClient, jxClient, kubeClient, ns, err := getClientsAndNamespace()
if err != nil {
return nil, err
}
return NewMetaPipelineClientWithClientsAndNamespace(jxClient, tektonClient, kubeClient, ns)
}
// NewMetaPipelineClientWithClientsAndNamespace creates a new client for the creation and application of meta pipelines using the specified parameters.
func NewMetaPipelineClientWithClientsAndNamespace(jxClient versioned.Interface, tektonClient tektonclient.Interface, kubeClient kubernetes.Interface, ns string) (Client, error) {
url, ref, err := versionStreamURLAndRef(jxClient, ns)
if err != nil {
return nil, errors.Wrap(err, "unable to determine versions stream URL and ref")
}
versionDir, err := cloneVersionStream(url, ref)
if err != nil {
return nil, errors.Wrap(err, "unable to clone version dir")
}
client := clientFactory{
jxClient: jxClient,
tektonClient: tektonClient,
kubeClient: kubeClient,
ns: ns,
versionDir: versionDir,
versionStreamURL: url,
versionStreamRef: ref,
}
return &client, nil
}
// Create creates the Tekton CRDs needed for executing the pipeline as defined by the input parameters.
func (c *clientFactory) Create(param PipelineCreateParam) (kube.PromoteStepActivityKey, tekton.CRDWrapper, error) {
err := c.cloneVersionStreamIfNeeded()
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, errors.Wrap(err, "unable to clone version stream")
}
gitInfo, err := gits.ParseGitURL(param.PullRef.SourceURL())
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, errors.Wrap(err, fmt.Sprintf("unable to determine needed git info from the specified git url '%s'", param.PullRef.SourceURL()))
}
podTemplates, err := c.getPodTemplates(apps.AppPodTemplateName)
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, errors.Wrap(err, "unable to retrieve pod templates")
}
branchIdentifier, err := c.determineBranchIdentifier(param.PipelineKind, param.PullRef)
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, errors.Wrap(err, "unable to create branch identifier")
}
pipelineName := tekton.PipelineResourceNameFromGitInfo(gitInfo, branchIdentifier, param.Context, tekton.MetaPipeline.String())
buildNumber, err := tekton.GenerateNextBuildNumber(c.tektonClient, c.jxClient, c.ns, gitInfo, branchIdentifier, retryDuration, param.Context, param.UseActivityForNextBuildNumber)
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, errors.Wrap(err, "unable to determine next build number")
}
logger.WithField("repo", gitInfo.URL).WithField("buildNumber", buildNumber).Debug("creating meta pipeline CRDs")
extendingApps, err := getExtendingApps(c.jxClient, c.ns)
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, err
}
crdCreationParams := CRDCreationParameters{
Namespace: c.ns,
Context: param.Context,
PipelineName: pipelineName,
PipelineKind: param.PipelineKind,
BuildNumber: buildNumber,
BranchIdentifier: branchIdentifier,
PullRef: param.PullRef,
SourceDir: defaultCheckoutDir,
PodTemplates: podTemplates,
ServiceAccount: param.ServiceAccount,
Labels: param.Labels,
EnvVars: param.EnvVariables,
DefaultImage: param.DefaultImage,
Apps: extendingApps,
VersionsDir: c.versionDir,
GitInfo: *gitInfo,
UseBranchAsRevision: param.UseBranchAsRevision,
NoReleasePrepare: param.NoReleasePrepare,
}
return c.createActualCRDs(buildNumber, branchIdentifier, param.Context, param.PullRef, crdCreationParams)
}
func (c *clientFactory) createActualCRDs(buildNumber string, branchIdentifier string, context string, pullRef PullRef, parameters CRDCreationParameters) (kube.PromoteStepActivityKey, tekton.CRDWrapper, error) {
tektonCRDs, err := createMetaPipelineCRDs(parameters)
if err != nil {
return kube.PromoteStepActivityKey{}, tekton.CRDWrapper{}, errors.Wrap(err, "failed to generate Tekton CRDs for meta pipeline")
}
pr, _ := tekton.ParsePullRefs(pullRef.String())
pipelineActivity := tekton.GeneratePipelineActivity(buildNumber, branchIdentifier, ¶meters.GitInfo, context, pr)
return *pipelineActivity, *tektonCRDs, nil
}
// Apply takes the given CRDs to process them, usually applying them to the cluster.
func (c *clientFactory) Apply(pipelineActivity kube.PromoteStepActivityKey, crds tekton.CRDWrapper) error {
err := tekton.ApplyPipeline(c.jxClient, c.kubeClient, c.tektonClient, &crds, c.ns, &pipelineActivity)
if err != nil {
return errors.Wrapf(err, "failed to apply Tekton CRDs")
}
logger.WithField("pipeline", crds.PipelineRun().Name).Debug("applied tekton CRDs")
return nil
}
// Close cleans up the resources use by this client.
func (c *clientFactory) Close() error {
return os.RemoveAll(c.versionDir)
}
func (c *clientFactory) getPodTemplates(containerName string) (map[string]*corev1.Pod, error) {
podTemplates, err := kube.LoadPodTemplates(c.kubeClient, c.ns)
if err != nil {
return nil, err
}
return podTemplates, nil
}
func (c *clientFactory) determineBranchIdentifier(pipelineType PipelineKind, pullRef PullRef) (string, error) {
var branch string
switch pipelineType {
case ReleasePipeline:
// no pull requests to merge, taking base branch name as identifier
branch = pullRef.baseBranch
case PullRequestPipeline:
if len(pullRef.pullRequests) == 0 {
return "", errors.New("pullrequest pipeline requested, but no pull requests specified")
}
branch = fmt.Sprintf("PR-%s", pullRef.PullRequests()[0].ID)
default:
branch = "unknown"
}
return branch, nil
}
func versionStreamURLAndRef(jxClient versioned.Interface, ns string) (string, string, error) {
devEnv, err := kube.GetDevEnvironment(jxClient, ns)
if err != nil {
return "", "", errors.Wrap(err, "unable to retrieve team environment")
}
if devEnv == nil {
return config.DefaultVersionsURL, config.DefaultVersionsRef, nil
}
teamSettings := devEnv.Spec.TeamSettings
url := teamSettings.VersionStreamURL
ref := teamSettings.VersionStreamRef
if url == "" {
url = config.DefaultVersionsURL
}
if ref == "" {
ref = config.DefaultVersionsRef
}
return url, ref, nil
}
func (c *clientFactory) cloneVersionStreamIfNeeded() error {
url, ref, err := versionStreamURLAndRef(c.jxClient, c.ns)
if err != nil {
return err
}
if c.versionStreamURL != url || c.versionStreamRef != ref {
oldVersionStreamDir := c.versionDir
c.versionDir, err = cloneVersionStream(url, ref)
if err != nil {
return err
}
_ = os.RemoveAll(oldVersionStreamDir)
}
return nil
}
func cloneVersionStream(url string, ref string) (string, error) {
dir, err := ioutil.TempDir("", "jx-version-repo-")
if err != nil {
return "", errors.Wrap(err, "unable to create temp dir for version stream")
}
logger.Debugf("cloning version stream url: %s ref: %s into %s", url, ref, dir)
// Not using GitCLi Clone/ShallowClone atm, since it does not work with tags.
// Once https://github.com/jenkins-x/jx/issues/5087 is resolved we should switch to that.
// As a quick hack is assumes that any ref with a '.' won't be a SHA.
if ref == "master" || strings.Contains(ref, ".") {
args := []string{"clone", "--depth", "1", "--branch", ref, url, "."}
cmd := util.Command{
Dir: dir,
Name: "git",
Args: args,
}
output, err := cmd.RunWithoutRetry()
if err != nil {
return "", errors.Wrapf(err, "unable to clone version stream and checking out branch/tag: %s", output)
}
} else {
// assuming we deal with a SHA
args := []string{"clone", url, "."}
cmd := util.Command{
Dir: dir,
Name: "git",
Args: args,
}
output, err := cmd.RunWithoutRetry()
if err != nil {
return "", errors.Wrapf(err, "unable to clone version stream: %s", output)
}
// Fetch PR refs before checking out the ref
args = []string{"fetch", "origin", ref}
cmd = util.Command{
Dir: dir,
Name: "git",
Args: args,
}
output, err = cmd.RunWithoutRetry()
if err != nil {
return "", errors.Wrapf(err, "unable to fetch pull request refs for version stream: %s", output)
}
args = []string{"checkout", ref}
cmd = util.Command{
Dir: dir,
Name: "git",
Args: args,
}
output, err = cmd.RunWithoutRetry()
if err != nil {
return "", errors.Wrapf(err, "unable checkout sha %s for version stream %s: %s", ref, url, output)
}
}
return dir, err
}
func getClientsAndNamespace() (tektonclient.Interface, jxclient.Interface, kubeclient.Interface, string, error) {
factory := jxfactory.NewFactory()
tektonClient, _, err := factory.CreateTektonClient()
if err != nil {
return nil, nil, nil, "", errors.Wrap(err, "unable to create Tekton client")
}
jxClient, _, err := factory.CreateJXClient()
if err != nil {
return nil, nil, nil, "", errors.Wrap(err, "unable to create JX client")
}
kubeClient, ns, err := factory.CreateKubeClient()
if err != nil {
return nil, nil, nil, "", errors.Wrap(err, "unable to create Kube client")
}
ns, _, err = kube.GetDevNamespace(kubeClient, ns)
if err != nil {
return nil, nil, nil, "", errors.Wrap(err, "unable to find the dev namespace")
}
return tektonClient, jxClient, kubeClient, ns, nil
}