-
Notifications
You must be signed in to change notification settings - Fork 51
/
apiclient.go
332 lines (272 loc) · 8.83 KB
/
apiclient.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
package apiclient
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"path"
"time"
"github.com/hashicorp/go-retryablehttp"
"github.com/italia/publiccode-crawler/v4/common"
internalUrl "github.com/italia/publiccode-crawler/v4/internal"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
type APIClient struct {
baseURL string
retryableClient *http.Client
token string
}
type Links struct {
Prev string `json:"prev"`
Next string `json:"next"`
}
type PublishersPaginated struct {
Data []Publisher `json:"data"`
Links Links `json:"links"`
}
type SoftwarePaginated struct {
Data []Software `json:"data"`
Links Links `json:"links"`
}
type Publisher struct {
ID string `json:"id"`
AlternativeID string `json:"alternativeId"`
Email string `json:"email"`
Description string `json:"description"`
CodeHostings []CodeHosting `json:"codeHosting"`
Active bool `json:"active"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type CodeHosting struct {
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
Group bool `json:"group"`
URL string `json:"url"`
}
type Software struct {
ID string `json:"id"`
URL string `json:"url"`
Aliases []string `json:"aliases"`
PubliccodeYml string `json:"publiccodeYml"`
Active bool `json:"active"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func NewClient() APIClient {
retryableClient := retryablehttp.NewClient().StandardClient()
return APIClient{
baseURL: viper.GetString("API_BASEURL"),
retryableClient: retryableClient,
token: "Bearer " + viper.GetString("API_BEARER_TOKEN"),
}
}
func (clt APIClient) Get(url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
if err != nil {
return nil, err
}
return clt.retryableClient.Do(req)
}
func (clt APIClient) Post(url string, body []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(
context.Background(),
http.MethodPost,
url,
bytes.NewBuffer(body),
)
if err != nil {
return nil, err
}
req.Header.Add("Authorization", clt.token)
req.Header.Add("Content-Type", "application/json")
return clt.retryableClient.Do(req)
}
func (clt APIClient) Patch(url string, body []byte) (*http.Response, error) {
req, err := http.NewRequestWithContext(
context.Background(),
http.MethodPatch,
url,
bytes.NewBuffer(body),
)
if err != nil {
return nil, err
}
req.Header.Add("Authorization", clt.token)
req.Header.Add("Content-Type", "application/merge-patch+json")
return clt.retryableClient.Do(req)
}
// GetPublishers returns a slice with all the publishers from the API and
// any error encountered.
func (clt APIClient) GetPublishers() ([]common.Publisher, error) {
var publishersResponse *PublishersPaginated
pageAfter := ""
publishers := make([]common.Publisher, 0, 25)
page:
reqURL := joinPath(clt.baseURL, "/publishers") + pageAfter
res, err := clt.Get(reqURL)
if err != nil {
return nil, fmt.Errorf("can't get publishers %s: %w", reqURL, err)
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, fmt.Errorf("can't get publishers %s: HTTP status %s", reqURL, res.Status)
}
publishersResponse = &PublishersPaginated{}
err = json.NewDecoder(res.Body).Decode(&publishersResponse)
if err != nil {
return nil, fmt.Errorf("can't parse GET %s response: %w", reqURL, err)
}
for _, p := range publishersResponse.Data {
var groups, repos []internalUrl.URL
for _, codeHosting := range p.CodeHostings {
u, err := url.Parse(codeHosting.URL)
if err != nil {
return nil, fmt.Errorf("can't parse GET %s response: %w", reqURL, err)
}
if codeHosting.Group {
groups = append(groups, (internalUrl.URL)(*u))
} else {
repos = append(repos, (internalUrl.URL)(*u))
}
}
id := p.ID
// Let's give precedence to the alternativeId. It's usually set by
// at Publisher creation and it's supposed to be more representative of the
// Publisher than the automatically generated UUID, since it's explicitly
// set with the API by the creator.
//
// This way, we also take a minimalist approach to Publisher concept in the crawler,
// having just one id.
if p.AlternativeID != "" {
id = p.ID
}
publishers = append(publishers, common.Publisher{
ID: id,
Name: fmt.Sprintf("%s %s", p.Description, p.Email),
Organizations: groups,
Repositories: repos,
})
}
if publishersResponse.Links.Next != "" {
pageAfter = publishersResponse.Links.Next
goto page
}
return publishers, nil
}
// GetSoftwareByURL returns the software matching the given repo URL and
// any error encountered.
// In case no software is found and no error occours, (nil, nil) is returned.
func (clt APIClient) GetSoftwareByURL(url string) (*Software, error) {
var softwareResponse SoftwarePaginated
res, err := clt.retryableClient.Get(joinPath(clt.baseURL, "/software") + "?url=" + url)
if err != nil {
return nil, fmt.Errorf("can't GET /software?url=%s: %w", url, err)
}
defer res.Body.Close()
err = json.NewDecoder(res.Body).Decode(&softwareResponse)
if err != nil {
return nil, fmt.Errorf("can't parse GET /software?url=%s response: %w", url, err)
}
if len(softwareResponse.Data) > 0 {
return &softwareResponse.Data[0], nil
}
return nil, nil //nolint:nilnil
}
// PostSoftware creates a new software resource with the given fields and returns
// a Software struct or any error encountered.
func (clt APIClient) PostSoftware(url string, aliases []string, publiccodeYml string, active bool) (*Software, error) {
body, err := json.Marshal(map[string]interface{}{
"publiccodeYml": publiccodeYml,
"url": url,
"aliases": aliases,
"active": active,
})
if err != nil {
return nil, fmt.Errorf("can't create software: %w", err)
}
res, err := clt.Post(joinPath(clt.baseURL, "/software"), body)
if err != nil {
return nil, fmt.Errorf("can't create software: %w", err)
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, fmt.Errorf("can't create software: API replied with HTTP %s", res.Status)
}
postSoftwareResponse := &Software{}
err = json.NewDecoder(res.Body).Decode(&postSoftwareResponse)
if err != nil {
return nil, fmt.Errorf("can't parse POST /software (for %s) response: %w", url, err)
}
return postSoftwareResponse, nil
}
// PatchSoftware updates a software resource with the given fields and returns
// an http.Response and any error encountered.
func (clt APIClient) PatchSoftware(
id string, url string, aliases []string, publiccodeYml string,
) (*http.Response, error) {
body, err := json.Marshal(map[string]interface{}{
"publiccodeYml": publiccodeYml,
"url": url,
"aliases": aliases,
})
if err != nil {
return nil, fmt.Errorf("can't update software: %w", err)
}
res, err := clt.Patch(joinPath(clt.baseURL, "/software/"+id), body)
if err != nil {
return res, fmt.Errorf("can't update software: %w", err)
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return res, fmt.Errorf("can't update software: API replied with HTTP %s", res.Status)
}
return res, nil
}
// PostSoftwareLog creates a new software log with the given fields and returns
// an http.Response and any error encountered.
func (clt APIClient) PostSoftwareLog(softwareID string, message string) (*http.Response, error) {
payload, err := json.Marshal(map[string]interface{}{
"message": message,
})
if err != nil {
return nil, fmt.Errorf("can't create log: %w", err)
}
res, err := clt.Post(joinPath(clt.baseURL, "/software/", softwareID, "logs"), payload)
if err != nil {
return res, fmt.Errorf("can't create software log: %w", err)
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return res, fmt.Errorf("can't create software log: API replied with HTTP %s", res.Status)
}
return res, nil
}
// PostLog creates a new log with the given message and returns an http.Response
// and any error encountered.
func (clt APIClient) PostLog(message string) (*http.Response, error) {
payload, err := json.Marshal(map[string]interface{}{
"message": message,
})
if err != nil {
return nil, fmt.Errorf("can't create log: %w", err)
}
res, err := clt.Post(joinPath(clt.baseURL, "/logs"), payload)
if err != nil {
return res, fmt.Errorf("can't create log: %w", err)
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return res, fmt.Errorf("can't create log: API replied with HTTP %s", res.Status)
}
return res, nil
}
func joinPath(base string, paths ...string) string {
u, err := url.Parse(base)
if err != nil {
log.Fatal(err)
}
for _, p := range paths {
u.Path = path.Join(u.Path, p)
}
return u.String()
}