-
Notifications
You must be signed in to change notification settings - Fork 786
/
install_requirements.go
987 lines (887 loc) · 40.3 KB
/
install_requirements.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
package config
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"reflect"
"strings"
"github.com/ghodss/yaml"
"github.com/imdario/mergo"
"github.com/pkg/errors"
"github.com/vrischmann/envconfig"
v1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1"
"github.com/jenkins-x/jx-logging/pkg/log"
"github.com/jenkins-x/jx/v2/pkg/cloud"
"github.com/jenkins-x/jx/v2/pkg/cloud/gke"
"github.com/jenkins-x/jx/v2/pkg/util"
)
var (
// autoDNSSuffixes the DNS suffixes of any auto-DNS services
autoDNSSuffixes = []string{
".nip.io",
".xip.io",
".beesdns.com",
}
)
const (
// DefaultFailOnValidationError by default fail if validation fails when reading jx-requirements
DefaultFailOnValidationError = true
)
const (
// RequirementsConfigFileName is the name of the requirements configuration file
RequirementsConfigFileName = "jx-requirements.yml"
// RequirementsValuesFileName is the name of the requirements configuration file
RequirementsValuesFileName = "jx-requirements.values.yaml.gotmpl"
// RequirementDomainIssuerUsername contains the username used for basic auth when requesting a domain
RequirementDomainIssuerUsername = "JX_REQUIREMENT_DOMAIN_ISSUER_USERNAME"
// RequirementDomainIssuerPassword contains the password used for basic auth when requesting a domain
RequirementDomainIssuerPassword = "JX_REQUIREMENT_DOMAIN_ISSUER_PASSWORD"
// RequirementDomainIssuerURL contains the URL to the service used when requesting a domain
RequirementDomainIssuerURL = "JX_REQUIREMENT_DOMAIN_ISSUER_URL"
// RequirementClusterName is the cluster name
RequirementClusterName = "JX_REQUIREMENT_CLUSTER_NAME"
// RequirementProject is the cloudprovider project
RequirementProject = "JX_REQUIREMENT_PROJECT"
// RequirementZone zone the cluster is in
RequirementZone = "JX_REQUIREMENT_ZONE"
// RequirementEnvGitOwner the default git owner for environment repositories if none is specified explicitly
RequirementEnvGitOwner = "JX_REQUIREMENT_ENV_GIT_OWNER"
// RequirementEnvGitPublic sets the visibility of the environment repositories as private (subscription required for GitHub Organisations)
RequirementEnvGitPublic = "JX_REQUIREMENT_ENV_GIT_PUBLIC"
// RequirementGitPublic sets the visibility of the application repositories as private (subscription required for GitHub Organisations)
RequirementGitPublic = "JX_REQUIREMENT_GIT_PUBLIC"
// RequirementExternalDNSServiceAccountName the service account name for external dns
RequirementExternalDNSServiceAccountName = "JX_REQUIREMENT_EXTERNALDNS_SA_NAME"
// RequirementVaultName the name for vault
RequirementVaultName = "JX_REQUIREMENT_VAULT_NAME"
// RequirementVaultServiceAccountName the service account name for vault
RequirementVaultServiceAccountName = "JX_REQUIREMENT_VAULT_SA_NAME"
// RequirementVeleroServiceAccountName the service account name for velero
RequirementVeleroServiceAccountName = "JX_REQUIREMENT_VELERO_SA_NAME"
// RequirementVeleroTTL defines the time to live (TTL) for the Velero backups in minutes
RequirementVeleroTTL = "JX_REQUIREMENT_VELERO_TTL"
// RequirementVeleroSchedule defines the schedule of the Velero backups in cron notation
RequirementVeleroSchedule = "JX_REQUIREMENT_VELERO_SCHEDULE"
// RequirementVaultKeyringName the keyring name for vault
RequirementVaultKeyringName = "JX_REQUIREMENT_VAULT_KEYRING_NAME"
// RequirementVaultKeyName the key name for vault
RequirementVaultKeyName = "JX_REQUIREMENT_VAULT_KEY_NAME"
// RequirementVaultBucketName the vault name for vault
RequirementVaultBucketName = "JX_REQUIREMENT_VAULT_BUCKET_NAME"
// RequirementVaultRecreateBucket recreate the bucket that vault uses
RequirementVaultRecreateBucket = "JX_REQUIREMENT_VAULT_RECREATE_BUCKET"
// RequirementVaultDisableURLDiscovery override the default lookup of the Vault URL, could be incluster service or external ingress
RequirementVaultDisableURLDiscovery = "JX_REQUIREMENT_VAULT_DISABLE_URL_DISCOVERY"
// RequirementSecretStorageType the secret storage type
RequirementSecretStorageType = "JX_REQUIREMENT_SECRET_STORAGE_TYPE"
// RequirementKanikoServiceAccountName the service account name for kaniko
RequirementKanikoServiceAccountName = "JX_REQUIREMENT_KANIKO_SA_NAME"
// RequirementKaniko if kaniko is required
RequirementKaniko = "JX_REQUIREMENT_KANIKO"
// RequirementIngressTLSProduction use the lets encrypt production server
RequirementIngressTLSProduction = "JX_REQUIREMENT_INGRESS_TLS_PRODUCTION"
// RequirementChartRepository the helm chart repository for jx
RequirementChartRepository = "JX_REQUIREMENT_CHART_REPOSITORY"
// RequirementRegistry the container registry for jx
RequirementRegistry = "JX_REQUIREMENT_REGISTRY"
// RequirementRepository the artifact repository for jx
RequirementRepository = "JX_REQUIREMENT_REPOSITORY"
// RequirementWebhook the webhook handler for jx
RequirementWebhook = "JX_REQUIREMENT_WEBHOOK"
// RequirementStorageBackupEnabled if backup storage is required
RequirementStorageBackupEnabled = "JX_REQUIREMENT_STORAGE_BACKUP_ENABLED"
// RequirementStorageBackupURL backup storage url
RequirementStorageBackupURL = "JX_REQUIREMENT_STORAGE_BACKUP_URL"
// RequirementStorageLogsEnabled if log storage is required
RequirementStorageLogsEnabled = "JX_REQUIREMENT_STORAGE_LOGS_ENABLED"
// RequirementStorageLogsURL logs storage url
RequirementStorageLogsURL = "JX_REQUIREMENT_STORAGE_LOGS_URL"
// RequirementStorageReportsEnabled if report storage is required
RequirementStorageReportsEnabled = "JX_REQUIREMENT_STORAGE_REPORTS_ENABLED"
// RequirementStorageReportsURL report storage url
RequirementStorageReportsURL = "JX_REQUIREMENT_STORAGE_REPORTS_URL"
// RequirementStorageRepositoryEnabled if repository storage is required
RequirementStorageRepositoryEnabled = "JX_REQUIREMENT_STORAGE_REPOSITORY_ENABLED"
// RequirementStorageRepositoryURL repository storage url
RequirementStorageRepositoryURL = "JX_REQUIREMENT_STORAGE_REPOSITORY_URL"
// RequirementGkeProjectNumber is the gke project number
RequirementGkeProjectNumber = "JX_REQUIREMENT_GKE_PROJECT_NUMBER"
// RequirementGitAppEnabled if the github app should be used for access tokens
RequirementGitAppEnabled = "JX_REQUIREMENT_GITHUB_APP_ENABLED"
// RequirementGitAppURL contains the URL to the github app
RequirementGitAppURL = "JX_REQUIREMENT_GITHUB_APP_URL"
// RequirementDevEnvApprovers contains the optional list of users to populate the dev env's OWNERS with
RequirementDevEnvApprovers = "JX_REQUIREMENT_DEV_ENV_APPROVERS"
// RequirementVersionsGitRef contains the git ref of the version stream
RequirementVersionsGitRef = "JX_REQUIREMENT_VERSIONS_GIT_REF"
)
const (
// BootDeployNamespace environment variable for deployment namespace
BootDeployNamespace = "DEPLOY_NAMESPACE"
)
// SecretStorageType is the type of storage used for secrets
type SecretStorageType string
const (
// SecretStorageTypeVault specifies that we use vault to store secrets
SecretStorageTypeVault SecretStorageType = "vault"
// SecretStorageTypeLocal specifies that we use the local file system in
// `~/.jx/localSecrets` to store secrets
SecretStorageTypeLocal SecretStorageType = "local"
)
// SecretStorageTypeValues the string values for the secret storage
var SecretStorageTypeValues = []string{"local", "vault"}
// WebhookType is the type of a webhook strategy
type WebhookType string
const (
// WebhookTypeNone if we have yet to define a webhook
WebhookTypeNone WebhookType = ""
// WebhookTypeProw specifies that we use prow for webhooks
// see: https://github.com/kubernetes/test-infra/tree/master/prow
WebhookTypeProw WebhookType = "prow"
// WebhookTypeLighthouse specifies that we use lighthouse for webhooks
// see: https://github.com/jenkins-x/lighthouse
WebhookTypeLighthouse WebhookType = "lighthouse"
// WebhookTypeJenkins specifies that we use jenkins webhooks
WebhookTypeJenkins WebhookType = "jenkins"
)
// WebhookTypeValues the string values for the webhook types
var WebhookTypeValues = []string{"jenkins", "lighthouse", "prow"}
// RepositoryType is the type of a repository we use to store artifacts (jars, tarballs, npm packages etc)
type RepositoryType string
const (
// RepositoryTypeUnknown if we have yet to configure a repository
RepositoryTypeUnknown RepositoryType = ""
// RepositoryTypeArtifactory if you wish to use Artifactory as the artifact repository
RepositoryTypeArtifactory RepositoryType = "artifactory"
// RepositoryTypeBucketRepo if you wish to use bucketrepo as the artifact repository. see https://github.com/jenkins-x/bucketrepo
RepositoryTypeBucketRepo RepositoryType = "bucketrepo"
// RepositoryTypeNone if you do not wish to install an artifact repository
RepositoryTypeNone RepositoryType = "none"
// RepositoryTypeNexus if you wish to use Sonatype Nexus as the artifact repository
RepositoryTypeNexus RepositoryType = "nexus"
)
// RepositoryTypeValues the string values for the repository types
var RepositoryTypeValues = []string{"none", "bucketrepo", "nexus", "artifactory"}
const (
// DefaultProfileFile location of profle config
DefaultProfileFile = "profile.yaml"
// OpenSourceProfile constant for OSS profile
OpenSourceProfile = "oss"
// CloudBeesProfile constant for CloudBees profile
CloudBeesProfile = "cloudbees"
)
// Overrideable at build time - see Makefile
var (
// DefaultVersionsURL default version stream url
DefaultVersionsURL = "https://github.com/jenkins-x/jenkins-x-versions.git"
// DefaultVersionsRef default version stream ref
DefaultVersionsRef = "master"
// DefaultBootRepository default git repo for boot
DefaultBootRepository = "https://github.com/jenkins-x/jenkins-x-boot-config.git"
// LatestVersionStringsBucket optional bucket name to search in for latest version strings
LatestVersionStringsBucket = ""
// BinaryDownloadBaseURL the base URL for downloading the binary from - will always have "VERSION/jx-OS-ARCH.EXTENSION" appended to it when used
BinaryDownloadBaseURL = "https://github.com/jenkins-x/jx/releases/download/v"
// TLSDocURL the URL presented by `jx step verify preinstall` for documentation on configuring TLS
TLSDocURL = "https://jenkins-x.io/docs/getting-started/setup/boot/#ingress"
)
// EnvironmentConfig configures the organisation and repository name of the git repositories for environments
type EnvironmentConfig struct {
// Key is the key of the environment configuration
Key string `json:"key,omitempty"`
// Owner is the git user or organisation for the repository
Owner string `json:"owner,omitempty"`
// Repository is the name of the repository within the owner
Repository string `json:"repository,omitempty"`
// GitServer is the URL of the git server
GitServer string `json:"gitServer,omitempty"`
// GitKind is the kind of git server (github, bitbucketserver etc)
GitKind string `json:"gitKind,omitempty"`
// Ingress contains ingress specific requirements
Ingress IngressConfig `json:"ingress,omitempty"`
// RemoteCluster specifies this environment runs on a remote cluster to the development cluster
RemoteCluster bool `json:"remoteCluster,omitempty"`
// PromotionStrategy what kind of promotion strategy to use
PromotionStrategy v1.PromotionStrategyType `json:"promotionStrategy,omitempty"`
// URLTemplate is the template to use for your environment's exposecontroller generated URLs
URLTemplate string `json:"urlTemplate,omitempty"`
}
// IngressConfig contains dns specific requirements
type IngressConfig struct {
// DNS is enabled
ExternalDNS bool `json:"externalDNS"`
// CloudDNSSecretName secret name which contains the service account for external-dns and cert-manager issuer to
// access the Cloud DNS service to resolve a DNS challenge
CloudDNSSecretName string `json:"cloud_dns_secret_name,omitempty"`
// Domain to expose ingress endpoints
Domain string `json:"domain"`
// IgnoreLoadBalancer if the nginx-controller LoadBalancer service should not be used to detect and update the
// domain if you are using a dynamic domain resolver like `.nip.io` rather than a real DNS configuration.
// With this flag enabled the `Domain` value will be used and never re-created based on the current LoadBalancer IP address.
IgnoreLoadBalancer bool `json:"ignoreLoadBalancer,omitempty"`
// Exposer the exposer used to expose ingress endpoints. Defaults to "Ingress"
Exposer string `json:"exposer,omitempty"`
// NamespaceSubDomain the sub domain expression to expose ingress. Defaults to ".jx."
NamespaceSubDomain string `json:"namespaceSubDomain"`
// TLS enable automated TLS using certmanager
TLS TLSConfig `json:"tls"`
// DomainIssuerURL contains a URL used to retrieve a Domain
DomainIssuerURL string `json:"domainIssuerURL,omitempty" envconfig:"JX_REQUIREMENT_DOMAIN_ISSUER_URL"`
}
// BuildPackConfig contains build pack info
type BuildPackConfig struct {
// Location contains location config
BuildPackLibrary *BuildPackLibrary `json:"buildPackLibrary,omitempty"`
}
// BuildPackLibrary contains buildpack location
type BuildPackLibrary struct {
// Name
Name string `json:"name,omitempty"`
// GitURL
GitURL string `json:"gitURL,omitempty"`
// GitRef
GitRef string `json:"gitRef,omitempty"`
}
// TLSConfig contains TLS specific requirements
type TLSConfig struct {
// TLS enabled
Enabled bool `json:"enabled"`
// Email address to register with services like LetsEncrypt
Email string `json:"email"`
// Production false uses self-signed certificates from the LetsEncrypt staging server, true enables the production
// server which incurs higher rate limiting https://letsencrypt.org/docs/rate-limits/
Production bool `json:"production"`
// SecretName the name of the secret which contains the TLS certificate
SecretName string `json:"secretName,omitempty"`
}
// JxInstallProfile contains the jx profile info
type JxInstallProfile struct {
InstallType string
}
// StorageEntryConfig contains dns specific requirements for a kind of storage
type StorageEntryConfig struct {
// Enabled if the storage is enabled
Enabled bool `json:"enabled"`
// URL the cloud storage bucket URL such as 'gs://mybucket' or 's3://foo' or `azblob://thingy'
// see https://jenkins-x.io/architecture/storage/
URL string `json:"url"`
}
// StorageConfig contains dns specific requirements
type StorageConfig struct {
// Logs for storing build logs
Logs StorageEntryConfig `json:"logs"`
// Tests for storing test results, coverage + code quality reports
Reports StorageEntryConfig `json:"reports"`
// Repository for storing repository artifacts
Repository StorageEntryConfig `json:"repository"`
// Backup for backing up kubernetes resource
Backup StorageEntryConfig `json:"backup"`
}
// AzureConfig contains Azure specific requirements
type AzureConfig struct {
// RegistrySubscription the registry subscription for defaulting the container registry.
// Not used if you specify a Registry explicitly
RegistrySubscription string `json:"registrySubscription,omitempty"`
}
// GKEConfig contains GKE specific requirements
type GKEConfig struct {
// ProjectNumber the unique project number GKE assigns to a project (required for workload identity).
ProjectNumber string `json:"projectNumber,omitempty" envconfig:"JX_REQUIREMENT_GKE_PROJECT_NUMBER"`
}
// ClusterConfig contains cluster specific requirements
type ClusterConfig struct {
// AzureConfig the azure specific configuration
AzureConfig *AzureConfig `json:"azure,omitempty"`
// ChartRepository the repository URL to deploy charts to
ChartRepository string `json:"chartRepository,omitempty" envconfig:"JX_REQUIREMENT_CHART_REPOSITORY"`
// GKEConfig the gke specific configuration
GKEConfig *GKEConfig `json:"gke,omitempty"`
// EnvironmentGitOwner the default git owner for environment repositories if none is specified explicitly
EnvironmentGitOwner string `json:"environmentGitOwner,omitempty" envconfig:"JX_REQUIREMENT_ENV_GIT_OWNER"`
// EnvironmentGitPublic determines whether jx boot create public or private git repos for the environments
EnvironmentGitPublic bool `json:"environmentGitPublic,omitempty" envconfig:"JX_REQUIREMENT_ENV_GIT_PUBLIC"`
// GitPublic determines whether jx boot create public or private git repos for the applications
GitPublic bool `json:"gitPublic,omitempty" envconfig:"JX_REQUIREMENT_GIT_PUBLIC"`
// Provider the kubernetes provider (e.g. gke)
Provider string `json:"provider,omitempty"`
// Namespace the namespace to install the dev environment
Namespace string `json:"namespace,omitempty"`
// ProjectID the cloud project ID e.g. on GCP
ProjectID string `json:"project,omitempty" envconfig:"JX_REQUIREMENT_PROJECT"`
// ClusterName the logical name of the cluster
ClusterName string `json:"clusterName,omitempty" envconfig:"JX_REQUIREMENT_CLUSTER_NAME"`
// VaultName the name of the vault if using vault for secrets
// Deprecated
VaultName string `json:"vaultName,omitempty"`
// Region the cloud region being used
Region string `json:"region,omitempty"`
// Zone the cloud zone being used
Zone string `json:"zone,omitempty" envconfig:"JX_REQUIREMENT_ZONE"`
// GitName is the name of the default git service
GitName string `json:"gitName,omitempty"`
// GitKind is the kind of git server (github, bitbucketserver etc)
GitKind string `json:"gitKind,omitempty"`
// GitServer is the URL of the git server
GitServer string `json:"gitServer,omitempty"`
// ExternalDNSSAName the service account name for external dns
ExternalDNSSAName string `json:"externalDNSSAName,omitempty" envconfig:"JX_REQUIREMENT_EXTERNALDNS_SA_NAME"`
// Registry the host name of the container registry
Registry string `json:"registry,omitempty" envconfig:"JX_REQUIREMENT_REGISTRY"`
// VaultSAName the service account name for vault
// Deprecated
VaultSAName string `json:"vaultSAName,omitempty"`
// KanikoSAName the service account name for kaniko
KanikoSAName string `json:"kanikoSAName,omitempty" envconfig:"JX_REQUIREMENT_KANIKO_SA_NAME"`
// HelmMajorVersion contains the major helm version number. Assumes helm 2.x with no tiller if no value specified
HelmMajorVersion string `json:"helmMajorVersion,omitempty"`
// DevEnvApprovers contains an optional list of approvers to populate the initial OWNERS file in the dev env repo
DevEnvApprovers []string `json:"devEnvApprovers,omitempty"`
// DockerRegistryOrg the default organisation used for container images
DockerRegistryOrg string `json:"dockerRegistryOrg,omitempty"`
// StrictPermissions lets you decide how to boot the cluster when it comes to permissions
// If it's false, cluster wide permissions will be used, normal, namespaced permissions will be used otherwise
// and extra steps will be necessary to get the cluster working
StrictPermissions bool `json:"strictPermissions,omitempty"`
}
// VaultConfig contains Vault configuration for Boot
type VaultConfig struct {
// Name the name of the Vault if using Jenkins X managed Vault instance.
// Cannot be used in conjunction with the URL attribute
Name string `json:"name,omitempty"`
Bucket string `json:"bucket,omitempty" envconfig:"JX_REQUIREMENT_VAULT_BUCKET_NAME"`
RecreateBucket bool `json:"recreateBucket,omitempty"`
Keyring string `json:"keyring,omitempty" envconfig:"JX_REQUIREMENT_VAULT_KEYRING_NAME"`
Key string `json:"key,omitempty" envconfig:"JX_REQUIREMENT_VAULT_KEY_NAME"`
// DisableURLDiscovery allows us to optionally override the default lookup of the Vault URL, could be incluster service or external ingress
DisableURLDiscovery bool `json:"disableURLDiscovery,omitempty"`
// AWSConfig describes the AWS specific configuration needed for the Vault Operator.
AWSConfig *VaultAWSConfig `json:"aws,omitempty"`
// AzureConfig describes the Azure specific configuration needed for the Vault Operator.
AzureConfig *VaultAzureConfig `json:"azure,omitempty"`
// URL specifies the URL of an Vault instance to use for secret storage.
// Needs to be specified together with the Service Account and namespace to use for connecting to Vault.
// This cannot be used in conjunction with the Name attribute.
URL string `json:"url,omitempty"`
// ServiceAccount is the name of the Kubernetes service account allowed to authenticate against Vault.
ServiceAccount string `json:"serviceAccount,omitempty" envconfig:"JX_REQUIREMENT_VAULT_SA_NAME"`
// Namespace of the Kubernetes service account allowed to authenticate against Vault.
Namespace string `json:"namespace,omitempty"`
// SecretEngineMountPoint is the secret engine mount point to be used for writing data into the KV engine of Vault.
// If not specified the 'secret' is used.
SecretEngineMountPoint string `json:"secretEngineMountPoint,omitempty"`
// KubernetesAuthPath is the auth path of used for this cluster
// If not specified the 'kubernetes' is used.
KubernetesAuthPath string `json:"kubernetesAuthPath,omitempty"`
}
// VaultAWSConfig contains all the Vault configuration needed by Vault to be deployed in AWS
type VaultAWSConfig struct {
VaultAWSUnsealConfig
AutoCreate bool `json:"autoCreate,omitempty"`
DynamoDBTable string `json:"dynamoDBTable,omitempty"`
DynamoDBRegion string `json:"dynamoDBRegion,omitempty"`
ProvidedIAMUsername string `json:"iamUserName,omitempty"`
}
// VaultAWSUnsealConfig contains references to existing AWS resources that can be used to install Vault
type VaultAWSUnsealConfig struct {
KMSKeyID string `json:"kmsKeyId,omitempty"`
KMSRegion string `json:"kmsRegion,omitempty"`
S3Bucket string `json:"s3Bucket,omitempty"`
S3Prefix string `json:"s3Prefix,omitempty"`
S3Region string `json:"s3Region,omitempty"`
}
// VaultAzureConfig contains all the Vault configuration needed by Vault to be deployed in Azure
type VaultAzureConfig struct {
TenantID string `json:"tenantId,omitempty"`
VaultName string `json:"vaultName,omitempty"`
KeyName string `json:"keyName,omitempty"`
StorageAccountName string `json:"storageAccountName,omitempty"`
ContainerName string `json:"containerName,omitempty"`
}
// UnmarshalJSON method handles the rename of EnvironmentGitPrivate to EnvironmentGitPublic.
func (t *ClusterConfig) UnmarshalJSON(data []byte) error {
// need a type alias to go into infinite loop
type Alias ClusterConfig
aux := &struct {
*Alias
}{
Alias: (*Alias)(t),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
var raw map[string]json.RawMessage
err := json.Unmarshal(data, &raw)
if err != nil {
return err
}
_, gitPublicSet := raw["environmentGitPublic"]
private, gitPrivateSet := raw["environmentGitPrivate"]
if gitPrivateSet && gitPublicSet {
return errors.New("found settings for EnvironmentGitPublic as well as EnvironmentGitPrivate in ClusterConfig, only EnvironmentGitPublic should be used")
}
if gitPrivateSet {
log.Logger().Warn("EnvironmentGitPrivate specified in Cluster EnvironmentGitPrivate is deprecated use EnvironmentGitPublic instead.")
privateString := string(private)
if privateString == "true" {
t.EnvironmentGitPublic = false
} else {
t.EnvironmentGitPublic = true
}
}
return nil
}
// VersionStreamConfig contains version stream config
type VersionStreamConfig struct {
// URL of the version stream to use
URL string `json:"url"`
// Ref of the version stream to use
Ref string `json:"ref" envconfig:"JX_REQUIREMENT_VERSIONS_GIT_REF"`
}
// VeleroConfig contains the configuration for velero
type VeleroConfig struct {
// Namespace the namespace to install velero into
Namespace string `json:"namespace,omitempty"`
// ServiceAccount the cloud service account used to run velero
ServiceAccount string `json:"serviceAccount,omitempty" envconfig:"JX_REQUIREMENT_VELERO_SA_NAME"`
// Schedule of backups
Schedule string `json:"schedule" envconfig:"JX_REQUIREMENT_VELERO_SCHEDULE"`
// TimeToLive period for backups to be retained
TimeToLive string `json:"ttl" envconfig:"JX_REQUIREMENT_VELERO_TTL"`
}
// AutoUpdateConfig contains auto update config
type AutoUpdateConfig struct {
// Enabled autoupdate
Enabled bool `json:"enabled"`
// Schedule cron of auto updates
Schedule string `json:"schedule"`
}
// GithubAppConfig contains github app config
type GithubAppConfig struct {
// Enabled this determines whether this install should use the jenkins x github app for access tokens
Enabled bool `json:"enabled" envconfig:"JX_REQUIREMENT_GITHUB_APP_ENABLED"`
// Schedule cron of the github app token refresher
Schedule string `json:"schedule,omitempty"`
// URL contains a URL to the github app
URL string `json:"url,omitempty" envconfig:"JX_REQUIREMENT_GITHUB_APP_URL"`
}
// RequirementsValues contains the logical installation requirements in the `jx-requirements.yml` file as helm values
type RequirementsValues struct {
// RequirementsConfig contains the logical installation requirements
RequirementsConfig *RequirementsConfig `json:"jxRequirements,omitempty"`
}
// UserNameEmailConfig contains the user name and email of a user (e.g. pipeline user)
type UserNameEmailConfig struct {
// Username the username of the user
Username string `json:"username,omitempty"`
// Email the email address of the user
Email string `json:"email,omitempty"`
}
// RequirementsConfig contains the logical installation requirements in the `jx-requirements.yml` file when
// installing, configuring or upgrading Jenkins X via `jx boot`
type RequirementsConfig struct {
// AutoUpdate contains auto update config
AutoUpdate AutoUpdateConfig `json:"autoUpdate,omitempty"`
// BootConfigURL contains the url to which the dev environment is associated with
BootConfigURL string `json:"bootConfigURL,omitempty"`
// BuildPackConfig contains custom build pack settings
BuildPacks *BuildPackConfig `json:"buildPacks,omitempty"`
// Cluster contains cluster specific requirements
Cluster ClusterConfig `json:"cluster"`
// Environments the requirements for the environments
Environments []EnvironmentConfig `json:"environments,omitempty"`
// GithubApp contains github app config
GithubApp *GithubAppConfig `json:"githubApp,omitempty"`
// GitOps if enabled we will setup a webhook in the boot configuration git repository so that we can
// re-run 'jx boot' when changes merge to the master branch
GitOps bool `json:"gitops,omitempty"`
// Indicates if we are using helmfile and helm 3 to spin up environments. This is currently an experimental
// feature flag used to implement better Multi-Cluster support. See https://github.com/jenkins-x/jx/issues/6442
Helmfile bool `json:"helmfile,omitempty"`
// Kaniko whether to enable kaniko for building docker images
Kaniko bool `json:"kaniko,omitempty"`
// Ingress contains ingress specific requirements
Ingress IngressConfig `json:"ingress"`
// PipelineUser the user name and email used for running pipelines
PipelineUser *UserNameEmailConfig `json:"pipelineUser,omitempty"`
// Repository specifies what kind of artifact repository you wish to use for storing artifacts (jars, tarballs, npm modules etc)
Repository RepositoryType `json:"repository,omitempty" envconfig:"JX_REQUIREMENT_REPOSITORY"`
// SecretStorage how should we store secrets for the cluster
SecretStorage SecretStorageType `json:"secretStorage,omitempty" envconfig:"JX_REQUIREMENT_SECRET_STORAGE_TYPE"`
// Storage contains storage requirements
Storage StorageConfig `json:"storage"`
// Terraform specifies if we are managing the kubernetes cluster and cloud resources with Terraform
Terraform bool `json:"terraform,omitempty"`
// Vault the configuration for vault
Vault VaultConfig `json:"vault,omitempty"`
// Velero the configuration for running velero for backing up the cluster resources
Velero VeleroConfig `json:"velero,omitempty"`
// VersionStream contains version stream info
VersionStream VersionStreamConfig `json:"versionStream"`
// Webhook specifies what engine we should use for webhooks
Webhook WebhookType `json:"webhook,omitempty"`
}
// NewRequirementsConfig creates a default configuration file
func NewRequirementsConfig() *RequirementsConfig {
return &RequirementsConfig{
SecretStorage: SecretStorageTypeLocal,
Webhook: WebhookTypeProw,
}
}
// LoadRequirementsConfig loads the configuration if there is a project configuration file
// if there is not a file called `jx-requirements.yml` in the given dir we will scan up the parent
// directories looking for the requirements file as we often run 'jx' steps in sub directories.
func LoadRequirementsConfig(dir string, failOnValidationErrors bool) (*RequirementsConfig, string, error) {
absolute, err := filepath.Abs(dir)
if err != nil {
return nil, "", errors.Wrap(err, "creating absolute path")
}
for absolute != "" && absolute != "." && absolute != "/" {
fileName := filepath.Join(absolute, RequirementsConfigFileName)
absolute = filepath.Dir(absolute)
exists, err := util.FileExists(fileName)
if err != nil {
return nil, "", err
}
if !exists {
continue
}
config, err := LoadRequirementsConfigFile(fileName, failOnValidationErrors)
return config, fileName, err
}
return nil, "", errors.New("jx-requirements.yml file not found")
}
// LoadRequirementsConfigFile loads a specific project YAML configuration file
func LoadRequirementsConfigFile(fileName string, failOnValidationErrors bool) (*RequirementsConfig, error) {
config := &RequirementsConfig{}
_, err := os.Stat(fileName)
if err != nil {
return nil, errors.Wrapf(err, "checking if file %s exists", fileName)
}
data, err := ioutil.ReadFile(fileName)
if err != nil {
return nil, fmt.Errorf("failed to load file %s due to %s", fileName, err)
}
validationErrors, err := util.ValidateYaml(config, data)
if err != nil {
return nil, fmt.Errorf("failed to validate YAML file %s due to %s", fileName, err)
}
if len(validationErrors) > 0 {
log.Logger().Warnf("validation failures in YAML file %s: %s", fileName, strings.Join(validationErrors, ", "))
if failOnValidationErrors {
return nil, fmt.Errorf("validation failures in YAML file %s:\n%s", fileName, strings.Join(validationErrors, "\n"))
}
}
err = yaml.Unmarshal(data, config)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal YAML file %s due to %s", fileName, err)
}
config.addDefaults()
config.handleDeprecation()
return config, nil
}
// GetRequirementsConfigFromTeamSettings reads the BootRequirements string from TeamSettings and unmarshals it
func GetRequirementsConfigFromTeamSettings(settings *v1.TeamSettings) (*RequirementsConfig, error) {
if settings == nil {
return nil, nil
}
// TeamSettings does not have a real value for BootRequirements, so this is probably not a boot cluster.
if settings.BootRequirements == "" {
return nil, nil
}
config := &RequirementsConfig{}
data := []byte(settings.BootRequirements)
validationErrors, err := util.ValidateYaml(config, data)
if err != nil {
return config, fmt.Errorf("failed to validate requirements from team settings due to %s", err)
}
if len(validationErrors) > 0 {
return config, fmt.Errorf("validation failures in requirements from team settings:\n%s", strings.Join(validationErrors, "\n"))
}
err = yaml.Unmarshal(data, config)
if err != nil {
return config, fmt.Errorf("failed to unmarshal requirements from team settings due to %s", err)
}
return config, nil
}
// IsEmpty returns true if this configuration is empty
func (c *RequirementsConfig) IsEmpty() bool {
empty := &RequirementsConfig{}
return reflect.DeepEqual(empty, c)
}
// SaveConfig saves the configuration file to the given project directory
func (c *RequirementsConfig) SaveConfig(fileName string) error {
c.handleDeprecation()
data, err := yaml.Marshal(c)
if err != nil {
return err
}
err = ioutil.WriteFile(fileName, data, util.DefaultWritePermissions)
if err != nil {
return errors.Wrapf(err, "failed to save file %s", fileName)
}
if c.Helmfile {
y := RequirementsValues{
RequirementsConfig: c,
}
data, err = yaml.Marshal(y)
if err != nil {
return err
}
err = ioutil.WriteFile(path.Join(path.Dir(fileName), RequirementsValuesFileName), data, util.DefaultWritePermissions)
if err != nil {
return errors.Wrapf(err, "failed to save file %s", RequirementsValuesFileName)
}
}
return nil
}
type environmentsSliceTransformer struct{}
// environmentsSliceTransformer.Transformer is handling the correct merge of two EnvironmentConfig slices
// so we can both append extra items and merge existing ones so we don't lose any data
func (t environmentsSliceTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
if typ == reflect.TypeOf([]EnvironmentConfig{}) {
return func(dst, src reflect.Value) error {
d := dst.Interface().([]EnvironmentConfig)
s := src.Interface().([]EnvironmentConfig)
if dst.CanSet() {
for i, v := range s {
value := v
if i > len(d)-1 {
d = append(d, value)
} else {
err := mergo.Merge(&d[i], &value, mergo.WithOverride)
if err != nil {
return errors.Wrap(err, "error merging EnvironmentConfig slices")
}
}
}
dst.Set(reflect.ValueOf(d))
}
return nil
}
}
return nil
}
// MergeSave attempts to merge the provided RequirementsConfig with the caller's data.
// It does so overriding values in the source struct with non-zero values from the provided struct
// it defines non-zero per property and not for a while embedded struct, meaning that nested properties
// in embedded structs should also be merged correctly.
// if a slice is added a transformer will be needed to handle correctly merging the contained values
func (c *RequirementsConfig) MergeSave(src *RequirementsConfig, requirementsFileName string) error {
err := mergo.Merge(c, src, mergo.WithOverride, mergo.WithTransformers(environmentsSliceTransformer{}))
if err != nil {
return errors.Wrap(err, "error merging jx-requirements.yml files")
}
err = c.SaveConfig(requirementsFileName)
if err != nil {
return errors.Wrapf(err, "error saving the merged jx-requirements.yml files to %s", requirementsFileName)
}
return nil
}
// EnvironmentMap creates a map of maps tree which can be used inside Go templates to access the environment
// configurations
func (c *RequirementsConfig) EnvironmentMap() map[string]interface{} {
answer := map[string]interface{}{}
for _, env := range c.Environments {
e := env
k := e.Key
if k == "" {
log.Logger().Warnf("missing 'key' for Environment requirements %#v", e)
continue
}
m, err := util.ToObjectMap(&e)
if err == nil {
ensureHasFields(m, "owner", "repository", "gitServer", "gitKind")
answer[k] = m
} else {
log.Logger().Warnf("failed to turn environment %s with value %#v into a map: %s\n", k, e, err.Error())
}
}
return answer
}
// Environment looks up the environment configuration based on environment name
func (c *RequirementsConfig) Environment(name string) (*EnvironmentConfig, error) {
for _, env := range c.Environments {
if env.Key == name {
return &env, nil
}
}
return nil, fmt.Errorf("environment %q not found", name)
}
// ToMap converts this object to a map of maps for use in helm templating
func (c *RequirementsConfig) ToMap() (map[string]interface{}, error) {
m, err := util.ToObjectMap(c)
if m != nil {
ensureHasFields(m, "provider", "project", "environmentGitOwner", "gitops", "webhook")
}
return m, err
}
// IsCloudProvider returns true if the kubenretes provider is a cloud
func (c *RequirementsConfig) IsCloudProvider() bool {
p := c.Cluster.Provider
return p == cloud.GKE || p == cloud.AKS || p == cloud.AWS || p == cloud.EKS || p == cloud.ALIBABA
}
func ensureHasFields(m map[string]interface{}, keys ...string) {
for _, k := range keys {
_, ok := m[k]
if !ok {
m[k] = ""
}
}
}
// MissingRequirement returns an error if there is a missing property in the requirements
func MissingRequirement(property string, fileName string) error {
return fmt.Errorf("missing property: %s in file %s", property, fileName)
}
// IsLazyCreateSecrets returns a boolean whether secrets should be lazily created
func (c *RequirementsConfig) IsLazyCreateSecrets(flag string) (bool, error) {
if flag != "" {
if flag == "true" {
return true, nil
} else if flag == "false" {
return false, nil
} else {
return false, util.InvalidOption("lazy-create", flag, []string{"true", "false"})
}
} else {
// lets default from the requirements
if !c.Terraform {
return true, nil
}
}
// default to false
return false, nil
}
// addDefaults lets ensure any missing values have good defaults
func (c *RequirementsConfig) addDefaults() {
if c.Cluster.Namespace == "" {
c.Cluster.Namespace = "jx"
}
if c.Cluster.GitServer == "" {
c.Cluster.GitServer = "https://github.com"
}
if c.Cluster.GitKind == "" {
c.Cluster.GitKind = "github"
}
if c.Cluster.GitName == "" {
c.Cluster.GitName = c.Cluster.GitKind
}
if c.Ingress.NamespaceSubDomain == "" {
c.Ingress.NamespaceSubDomain = "-" + c.Cluster.Namespace + "."
}
if c.Webhook == WebhookTypeNone {
if c.Cluster.GitServer == "https://github.com" || c.Cluster.GitServer == "https://github.com/" {
c.Webhook = WebhookTypeProw
} else {
// TODO when lighthouse is GA lets default to it
// c.Webhook = WebhookTypeLighthouse
}
}
if c.Repository == "" {
c.Repository = "nexus"
}
}
func (c *RequirementsConfig) handleDeprecation() {
if c.Vault.Name != "" {
c.Cluster.VaultName = c.Vault.Name
} else {
c.Vault.Name = c.Cluster.VaultName
}
if c.Vault.ServiceAccount != "" {
c.Cluster.VaultSAName = c.Vault.ServiceAccount
} else {
c.Vault.ServiceAccount = c.Cluster.VaultSAName
}
}
// IsAutoDNSDomain returns true if the domain is configured to use an auto DNS sub domain like
// '.nip.io' or '.xip.io'
func (i *IngressConfig) IsAutoDNSDomain() bool {
for _, suffix := range autoDNSSuffixes {
if strings.HasSuffix(i.Domain, suffix) {
return true
}
}
return false
}
// OverrideRequirementsFromEnvironment allows properties to be overridden with environment variables
func (c *RequirementsConfig) OverrideRequirementsFromEnvironment(gcloudFn func() gke.GClouder) {
// struct members need to use explicit 'envconfig:"<var-name>"' unless there is a match between struct member navigation
// path and env variable name
err := envconfig.InitWithOptions(&c, envconfig.Options{AllOptional: true, LeaveNil: true, Prefix: "JX_REQUIREMENT"})
if err != nil {
log.Logger().Errorf("Unable to init envconfig for override requirements: %s", err)
}
// RequirementIngressTLSProduction applies to more than one environment and needs to be handled explicitly
if "" != os.Getenv(RequirementIngressTLSProduction) {
useProduction := os.Getenv(RequirementIngressTLSProduction)
if envVarBoolean(useProduction) {
c.Ingress.TLS.Production = true
for _, e := range c.Environments {
e.Ingress.TLS.Production = true
}
} else {
c.Ingress.TLS.Production = false
for _, e := range c.Environments {
e.Ingress.TLS.Production = false
}
}
}
// StorageConfig is reused between multiple storage configuration types and needs to be handled explicitly
if "" != os.Getenv(RequirementStorageBackupEnabled) {
storageBackup := os.Getenv(RequirementStorageBackupEnabled)
if envVarBoolean(storageBackup) && "" != os.Getenv(RequirementStorageBackupURL) {
c.Storage.Backup.Enabled = true
c.Storage.Backup.URL = os.Getenv(RequirementStorageBackupURL)
}
}
if "" != os.Getenv(RequirementStorageLogsEnabled) {
storageLogs := os.Getenv(RequirementStorageLogsEnabled)
if envVarBoolean(storageLogs) && "" != os.Getenv(RequirementStorageLogsURL) {
c.Storage.Logs.Enabled = true
c.Storage.Logs.URL = os.Getenv(RequirementStorageLogsURL)
}
}
if "" != os.Getenv(RequirementStorageReportsEnabled) {
storageReports := os.Getenv(RequirementStorageReportsEnabled)
if envVarBoolean(storageReports) && "" != os.Getenv(RequirementStorageReportsURL) {
c.Storage.Reports.Enabled = true
c.Storage.Reports.URL = os.Getenv(RequirementStorageReportsURL)
}
}
if "" != os.Getenv(RequirementStorageRepositoryEnabled) {
storageRepository := os.Getenv(RequirementStorageRepositoryEnabled)
if envVarBoolean(storageRepository) && "" != os.Getenv(RequirementStorageRepositoryURL) {
c.Storage.Repository.Enabled = true
c.Storage.Repository.URL = os.Getenv(RequirementStorageRepositoryURL)
}
}
if "" != os.Getenv(RequirementDevEnvApprovers) {
rawApprovers := os.Getenv(RequirementDevEnvApprovers)
for _, approver := range strings.Split(rawApprovers, ",") {
c.Cluster.DevEnvApprovers = append(c.Cluster.DevEnvApprovers, strings.TrimSpace(approver))
}
}
// set this if its not currently configured
if c.Cluster.Provider == "gke" {
if c.Cluster.GKEConfig == nil {
c.Cluster.GKEConfig = &GKEConfig{}
}
if c.Cluster.GKEConfig.ProjectNumber == "" {
if gcloudFn != nil {
gcloud := gcloudFn()
if gcloud != nil {
projectNumber, err := gcloud.GetProjectNumber(c.Cluster.ProjectID)
if err != nil {
log.Logger().Warnf("unable to determine gke project number - %s", err)
}
c.Cluster.GKEConfig.ProjectNumber = projectNumber
}
}
}
}
}
func envVarBoolean(value string) bool {
return value == "true" || value == "yes"
}