/
deploy.go
243 lines (201 loc) · 9.38 KB
/
deploy.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
package deploy
// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.
import (
"context"
"encoding/json"
"reflect"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/jongio/azidext/go/azidext"
"github.com/sirupsen/logrus"
"github.com/Azure/ARO-RP/pkg/deploy/vmsscleaner"
"github.com/Azure/ARO-RP/pkg/env"
"github.com/Azure/ARO-RP/pkg/util/arm"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/compute"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/dns"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/msi"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network"
"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage"
"github.com/Azure/ARO-RP/pkg/util/keyvault"
)
var _ Deployer = (*deployer)(nil)
type Deployer interface {
PreDeploy(context.Context) error
DeployRP(context.Context) error
DeployGateway(context.Context) error
UpgradeRP(context.Context) error
UpgradeGateway(context.Context) error
SaveVersion(context.Context) error
}
type deployer struct {
log *logrus.Entry
env env.Core
globaldeployments features.DeploymentsClient
globalgroups features.ResourceGroupsClient
globalrecordsets dns.RecordSetsClient
globalaccounts storage.AccountsClient
deployments features.DeploymentsClient
groups features.ResourceGroupsClient
loadbalancers network.LoadBalancersClient
userassignedidentities msi.UserAssignedIdentitiesClient
providers features.ProvidersClient
publicipaddresses network.PublicIPAddressesClient
resourceskus compute.ResourceSkusClient
roleassignments authorization.RoleAssignmentsClient
vmss compute.VirtualMachineScaleSetsClient
vmssvms compute.VirtualMachineScaleSetVMsClient
zones dns.ZonesClient
clusterKeyvault keyvault.Manager
dbtokenKeyvault keyvault.Manager
portalKeyvault keyvault.Manager
serviceKeyvault keyvault.Manager
config *RPConfig
version string
vmssCleaner vmsscleaner.Interface
}
// KnownDeploymentErrorType represents a type of error we encounter during an
// RP/gateway deployment that we know how to handle via automation.
type KnownDeploymentErrorType string
const (
KnownDeploymentErrorTypeRPLBNotFound KnownDeploymentErrorType = "RPLBNotFound"
)
// New initiates new deploy utility object
func New(ctx context.Context, log *logrus.Entry, _env env.Core, config *RPConfig, version string, tokenCredential azcore.TokenCredential) (Deployer, error) {
err := config.validate()
if err != nil {
return nil, err
}
scopes := []string{_env.Environment().ResourceManagerScope}
authorizer := azidext.NewTokenCredentialAdapter(tokenCredential, scopes)
scopes = []string{_env.Environment().KeyVaultScope}
kvAuthorizer := azidext.NewTokenCredentialAdapter(tokenCredential, scopes)
vmssClient := compute.NewVirtualMachineScaleSetsClient(_env.Environment(), config.SubscriptionID, authorizer)
return &deployer{
log: log,
env: _env,
globaldeployments: features.NewDeploymentsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer),
globalgroups: features.NewResourceGroupsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer),
globalrecordsets: dns.NewRecordSetsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer),
globalaccounts: storage.NewAccountsClient(_env.Environment(), *config.Configuration.GlobalSubscriptionID, authorizer),
deployments: features.NewDeploymentsClient(_env.Environment(), config.SubscriptionID, authorizer),
groups: features.NewResourceGroupsClient(_env.Environment(), config.SubscriptionID, authorizer),
loadbalancers: network.NewLoadBalancersClient(_env.Environment(), config.SubscriptionID, authorizer),
userassignedidentities: msi.NewUserAssignedIdentitiesClient(_env.Environment(), config.SubscriptionID, authorizer),
providers: features.NewProvidersClient(_env.Environment(), config.SubscriptionID, authorizer),
roleassignments: authorization.NewRoleAssignmentsClient(_env.Environment(), config.SubscriptionID, authorizer),
resourceskus: compute.NewResourceSkusClient(_env.Environment(), config.SubscriptionID, authorizer),
publicipaddresses: network.NewPublicIPAddressesClient(_env.Environment(), config.SubscriptionID, authorizer),
vmss: vmssClient,
vmssvms: compute.NewVirtualMachineScaleSetVMsClient(_env.Environment(), config.SubscriptionID, authorizer),
zones: dns.NewZonesClient(_env.Environment(), config.SubscriptionID, authorizer),
clusterKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.ClusterKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"),
dbtokenKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.DBTokenKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"),
portalKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.PortalKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"),
serviceKeyvault: keyvault.NewManager(kvAuthorizer, "https://"+*config.Configuration.KeyvaultPrefix+env.ServiceKeyvaultSuffix+"."+_env.Environment().KeyVaultDNSSuffix+"/"),
config: config,
version: version,
vmssCleaner: vmsscleaner.New(log, vmssClient),
}, nil
}
// getParameters returns an *arm.Parameters populated with parameter names and
// values. The names are taken from the ps argument and the values are taken
// from d.config.Configuration.
func (d *deployer) getParameters(ps map[string]interface{}) *arm.Parameters {
m := map[string]interface{}{}
v := reflect.ValueOf(*d.config.Configuration)
for i := 0; i < v.NumField(); i++ {
if v.Field(i).IsNil() {
continue
}
m[strings.SplitN(v.Type().Field(i).Tag.Get("json"), ",", 2)[0]] = v.Field(i).Interface()
}
parameters := &arm.Parameters{
Parameters: map[string]*arm.ParametersParameter{},
}
for p := range ps {
// do not convert empty fields
// makes default values templates work
v, ok := m[p]
if !ok {
continue
}
switch p {
case "gatewayDomains", "gatewayFeatures", "portalAccessGroupIds", "portalElevatedGroupIds", "rpFeatures":
v = strings.Join(v.([]string), ",")
}
parameters.Parameters[p] = &arm.ParametersParameter{
Value: v,
}
}
return parameters
}
func (d *deployer) deploy(ctx context.Context, rgName, deploymentName, vmssName string, deployment mgmtfeatures.Deployment) (err error) {
numAttempts := 3
for i := 0; i < numAttempts; i++ {
d.log.Printf("deploying %s", deploymentName)
err = d.deployments.CreateOrUpdateAndWait(ctx, rgName, deploymentName, deployment)
serviceErr, isServiceError := err.(*azure.ServiceError)
// As long as this is not the final deployment attempt,
// unconditionally log the error before inspecting it.
if err != nil && i < numAttempts-1 {
d.log.Print(err)
}
// Check for a known error that we know how to handle.
if isServiceError {
errorType, checkTypeErr := d.checkForKnownError(serviceErr, i)
if checkTypeErr != nil {
d.log.Printf("Encountered an error in checkForKnownError: %s", checkTypeErr)
}
// On new RP deployments, we get a spurious DeploymentFailed error
// from the Microsoft.Insights/metricAlerts resources indicating
// that rp-lb can't be found, even though it exists and the
// resources correctly have a dependsOn stanza referring to it.
// Retry once, and only if this error is encountered on the first
// deployment attempt.
if errorType == KnownDeploymentErrorTypeRPLBNotFound {
d.log.Print("Deployment encountered known ResourceNotFound error for RP LB; retrying.")
continue
}
}
// For errors we don't know how to handle, delete the failed VMSS and retry the deployment.
if err != nil && *d.config.Configuration.VMSSCleanupEnabled {
if retry := d.vmssCleaner.RemoveFailedNewScaleset(ctx, rgName, vmssName); retry {
continue
}
}
break
}
return err
}
// checkForKnownError is a helper function that checks the errors nested within an Azure ServiceError
// for a known error and returns the corresponding KnownDeploymentErrorType if applicable.
func (d *deployer) checkForKnownError(serviceErr *azure.ServiceError, deployAttempt int) (KnownDeploymentErrorType, error) {
if serviceErr.Code != "DeploymentFailed" || len(serviceErr.Details) == 0 {
return "", nil
}
outerErr := azure.ServiceError{}
jsonEncoded, err := json.Marshal(serviceErr.Details[0])
if err != nil {
return "", err
}
err = json.Unmarshal(jsonEncoded, &outerErr)
if err != nil {
return "", err
}
innerErr := azure.ServiceError{}
err = json.Unmarshal([]byte(outerErr.Message), &innerErr)
if err != nil {
return "", err
}
isFirstAttempt := deployAttempt < 1
isRPLBNotFound := innerErr.Code == "ResourceNotFound" && strings.Contains(innerErr.Message, "Microsoft.Network/loadBalancers/rp-lb")
if isFirstAttempt && isRPLBNotFound {
return KnownDeploymentErrorTypeRPLBNotFound, nil
}
return "", nil
}