-
Notifications
You must be signed in to change notification settings - Fork 55
/
remoteurlchecker.go
239 lines (209 loc) · 8.99 KB
/
remoteurlchecker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
package precheckrunner
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/artifactory/services"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/httputils"
"github.com/jfrog/jfrog-client-go/utils/log"
)
type RemoteUrlCheckStatus string
const (
remoteUrlCheckName = "Remote repositories URL connectivity"
remoteUrlCheckPollingTimeout = 30 * time.Minute
remoteUrlCheckPollingInterval = 5 * time.Second
remoteUrlCheckRetries = 3
remoteUrlCheckIntervalMilliSecs = 10000
)
type remoteRepoSettings struct {
Key string `json:"key,omitempty"`
Url string `json:"url,omitempty"`
RepoType string `json:"repo_type,omitempty"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
QueryParams string `json:"query_params,omitempty"`
}
type remoteUrlResponse struct {
Status RemoteUrlCheckStatus `json:"status,omitempty"`
InaccessibleRepositories []inaccessibleRepository `json:"inaccessible_repositories,omitempty"`
CheckedRepositories uint `json:"checked_repositories,omitempty"`
TotalRepositories uint `json:"total_repositories,omitempty"`
}
type inaccessibleRepository struct {
RepoKey string `json:"repo_key,omitempty"`
StatusCode int `json:"status_code,omitempty"`
Reason string `json:"reason,omitempty"`
Url string `json:"url,omitempty"`
}
// Run remote repository URLs accessibility test before transferring configuration from one Artifactory to another
type RemoteRepositoryCheck struct {
targetServicesManager *artifactory.ArtifactoryServicesManager
remoteRepositories []interface{}
}
func NewRemoteRepositoryCheck(targetServicesManager *artifactory.ArtifactoryServicesManager, remoteRepositories []interface{}) *RemoteRepositoryCheck {
return &RemoteRepositoryCheck{targetServicesManager, remoteRepositories}
}
func (rrc *RemoteRepositoryCheck) Name() string {
return remoteUrlCheckName
}
func (rrc *RemoteRepositoryCheck) ExecuteCheck(args RunArguments) (passed bool, err error) {
remoteUrlRequest, err := rrc.createRemoteUrlRequest()
if err != nil {
return false, err
}
inaccessibleRepositories, err := rrc.doCheckRemoteRepositories(args, remoteUrlRequest)
if err != nil {
return false, err
}
if len(*inaccessibleRepositories) == 0 {
return true, nil
}
return false, handleFailureRun(*inaccessibleRepositories)
}
// Create the remote URL request from the received remote repository details from Artifactory
func (rrc *RemoteRepositoryCheck) createRemoteUrlRequest() ([]remoteRepoSettings, error) {
remoteUrlRequests := make([]remoteRepoSettings, len(rrc.remoteRepositories))
for i, remoteRepository := range rrc.remoteRepositories {
// The remote repository interface is not necessarily of RemoteRepositoryBaseParams
// type (can be a map) and therefore we marshal and unmarshal it.
remoteRepositoryBytes, err := json.Marshal(remoteRepository)
if err != nil {
return nil, errorutils.CheckError(err)
}
var remoteRepositoryParams services.RemoteRepositoryBaseParams
if err = json.Unmarshal(remoteRepositoryBytes, &remoteRepositoryParams); err != nil {
return nil, errorutils.CheckError(err)
}
remoteUrlRequests[i] = remoteRepoSettings{
Key: remoteRepositoryParams.Key,
Url: remoteRepositoryParams.Url,
RepoType: remoteRepositoryParams.PackageType,
Username: remoteRepositoryParams.Username,
Password: remoteRepositoryParams.Password,
QueryParams: remoteRepositoryParams.QueryParams,
}
}
return remoteUrlRequests, nil
}
func (rrc *RemoteRepositoryCheck) doCheckRemoteRepositories(args RunArguments, remoteUrlRequest []remoteRepoSettings) (inaccessibleRepositories *[]inaccessibleRepository, err error) {
artifactoryUrl := clientutils.AddTrailingSlashIfNeeded(args.ServerDetails.ArtifactoryUrl)
body, err := json.Marshal(remoteUrlRequest)
if err != nil {
return nil, errorutils.CheckError(err)
}
// Create rtDetails
rtDetails, err := utils.CreateArtifactoryClientDetails(*rrc.targetServicesManager)
if err != nil {
return nil, err
}
progressBar, err := rrc.startCheckRemoteRepositories(rtDetails, artifactoryUrl, args, body)
if err != nil {
return nil, err
}
defer func() {
if progressBar != nil {
progressBar.GetBar().Abort(true)
}
}()
// Wait for remote repositories check completion
return rrc.waitForRemoteReposCheckCompletion(rtDetails, artifactoryUrl, progressBar)
}
func (rrc *RemoteRepositoryCheck) startCheckRemoteRepositories(rtDetails *httputils.HttpClientDetails, artifactoryUrl string, args RunArguments, requestBody []byte) (*progressbar.TasksProgressBar, error) {
var response *remoteUrlResponse
// Sometimes, POST api/plugins/execute/remoteRepositoriesCheck returns unexpectedly 404 errors, although the config-import plugin is installed.
// To overcome this issue, we use a custom retryExecutor and not the default retry executor that retries only on HTTP errors >= 500.
retryExecutor := clientutils.RetryExecutor{
Context: args.Context,
MaxRetries: remoteUrlCheckRetries,
RetriesIntervalMilliSecs: remoteUrlCheckIntervalMilliSecs,
ErrorMessage: fmt.Sprintf("Failed to start the remote repositories check in %s", artifactoryUrl),
LogMsgPrefix: "[Config import]",
ExecutionHandler: func() (shouldRetry bool, err error) {
// Start the remote repositories check process
resp, responseBody, err := (*rrc.targetServicesManager).Client().SendPost(artifactoryUrl+utils.PluginsExecuteRestApi+"remoteRepositoriesCheck", requestBody, rtDetails)
if err != nil {
return false, err
}
if err = errorutils.CheckResponseStatusWithBody(resp, responseBody, http.StatusOK); err != nil {
return true, err
}
response, err = unmarshalRemoteUrlResponse(responseBody)
return false, err
},
}
if err := retryExecutor.Execute(); err != nil {
return nil, err
}
if args.ProgressMng == nil {
return nil, nil
}
return args.ProgressMng.NewTasksProgressBar(int64(response.TotalRepositories), coreutils.IsWindows(), "Remote repositories"), nil
}
func (rrc *RemoteRepositoryCheck) waitForRemoteReposCheckCompletion(rtDetails *httputils.HttpClientDetails, artifactoryUrl string, progressBar *progressbar.TasksProgressBar) (*[]inaccessibleRepository, error) {
pollingExecutor := &httputils.PollingExecutor{
Timeout: remoteUrlCheckPollingTimeout,
PollingInterval: remoteUrlCheckPollingInterval,
MsgPrefix: "Waiting for remote repositories check completion in Artifactory server at " + artifactoryUrl,
PollingAction: rrc.createImportPollingAction(rtDetails, artifactoryUrl, progressBar),
}
body, err := pollingExecutor.Execute()
if err != nil {
return nil, err
}
response, err := unmarshalRemoteUrlResponse(body)
if err != nil {
return nil, err
}
return &response.InaccessibleRepositories, nil
}
func (rrc *RemoteRepositoryCheck) createImportPollingAction(rtDetails *httputils.HttpClientDetails, artifactoryUrl string, progressBar *progressbar.TasksProgressBar) httputils.PollingAction {
return func() (shouldStop bool, responseBody []byte, err error) {
// Get config import status
resp, body, _, err := (*rrc.targetServicesManager).Client().SendGet(artifactoryUrl+utils.PluginsExecuteRestApi+"remoteRepositoriesCheckStatus", true, rtDetails)
if err != nil {
return true, nil, err
}
// 200 - Import completed
if resp.StatusCode == http.StatusOK {
return true, body, nil
}
// 202 - Update status
if resp.StatusCode == http.StatusAccepted {
response, err := unmarshalRemoteUrlResponse(body)
if err != nil {
return true, nil, err
}
if progressBar != nil {
delta := int64(response.CheckedRepositories) - progressBar.GetBar().Current()
progressBar.GetBar().IncrInt64(delta)
}
}
return false, nil, nil
}
}
// Unmarshal response from Artifactory to remoteUrlResponse
func unmarshalRemoteUrlResponse(body []byte) (*remoteUrlResponse, error) {
log.Debug(fmt.Sprintf("Response from Artifactory:\n%s", body))
var response remoteUrlResponse
err := json.Unmarshal(body, &response)
return &response, errorutils.CheckError(err)
}
// Create csv summary of all the files with inaccessible remote repositories and log the result
func handleFailureRun(inaccessibleRepositories []inaccessibleRepository) (err error) {
// Create summary
csvPath, err := utils.CreateCSVFile("inaccessible-repositories", inaccessibleRepositories, time.Now())
if err != nil {
log.Error("Couldn't create the inaccessible remote repository URLs CSV file", err)
return
}
// Log result
log.Info(fmt.Sprintf("Found %d inaccessible remote repository URLs. Check the summary CSV file in: %s", len(inaccessibleRepositories), csvPath))
return
}