-
Notifications
You must be signed in to change notification settings - Fork 18
/
custom_reg_fields.go
289 lines (260 loc) · 9.42 KB
/
custom_reg_fields.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
/***************************************************************
*
* Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You may
* obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
***************************************************************/
package registry
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/jellydator/ttlcache/v3"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"github.com/pelicanplatform/pelican/config"
"github.com/pelicanplatform/pelican/param"
"github.com/pelicanplatform/pelican/server_structs"
"github.com/pelicanplatform/pelican/utils"
)
type (
customRegFieldsConfig struct {
Name string `mapstructure:"name"`
Type string `mapstructure:"type"`
Required bool `mapstructure:"required"`
Options []registrationFieldOption `mapstructure:"options"`
Description string `mapstructure:"description"`
OptionsUrl string `mapstructure:"optionsUrl"`
}
)
var (
customRegFieldsConfigs []customRegFieldsConfig
optionsCache = ttlcache.New(
ttlcache.WithTTL[string, []registrationFieldOption](5 * time.Minute),
)
)
func InitOptionsCache(ctx context.Context, egrp *errgroup.Group) {
go optionsCache.Start()
egrp.Go(func() error {
<-ctx.Done()
optionsCache.DeleteAll()
optionsCache.Stop()
return nil
})
}
func optionsToString(options []registrationFieldOption) (result string) {
for _, opt := range options {
result += fmt.Sprintf("ID: %s | Name: %s\n", opt.ID, opt.Name)
}
return
}
// Fetch from the optionsUrl, check the returned options, and set the optionsCache
func getCachedOptions(key string, ttl time.Duration) ([]registrationFieldOption, error) {
if optionsCache.Has(key) {
return optionsCache.Get(key).Value(), nil
}
// Fetch from URL
if key == "" {
return nil, errors.New("key is empty")
}
_, err := url.Parse(key)
if err != nil {
return nil, errors.Wrap(err, "key is not a valid URL")
}
client := http.Client{Transport: config.GetTransport()}
req, err := http.NewRequest(http.MethodGet, key, nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to create a new request for fetching key %s", key)
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
return nil, errors.Wrapf(err, "failed to request the key %s", key)
}
resBody, err := io.ReadAll(res.Body)
if err != nil {
return nil, errors.Wrapf(err, "failed to read the response body")
}
if res.StatusCode != 200 {
return nil, fmt.Errorf("fetching key %s returns status code %d with response body %s", key, res.StatusCode, resBody)
}
options := []registrationFieldOption{}
err = json.Unmarshal(resBody, &options)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse response from key %s to options struct with response body: %s", key, resBody)
}
isUnique := checkUniqueOptions(options)
if !isUnique {
return nil, fmt.Errorf("returned options from key %s are not unique. Options: %s", key, optionsToString(options))
}
// Check IDs are not empty
invalidName := ""
for _, opt := range options {
if opt.ID == "" {
invalidName = opt.Name
break
}
}
if invalidName != "" {
return nil, fmt.Errorf("returned options from key %s have empty ID for option %s", key, invalidName)
}
optionsCache.Set(key, options, ttl)
return options, nil
}
// Given the custom registration fields read from the config,
// convert them to an array of registrationField for web UI
func convertCustomRegFields(configFields []customRegFieldsConfig) []registrationField {
regFields := make([]registrationField, 0)
for _, field := range configFields {
optionsUrl := field.OptionsUrl
options := field.Options
if field.Type == string(Enum) {
if len(options) != 0 { // Options overwrites OptionsUrl
optionsUrl = ""
}
if optionsUrl != "" { // field.Options is not set but OptionsUrl is set
fetchedOptions, err := getCachedOptions(optionsUrl, ttlcache.DefaultTTL)
if err != nil {
log.Errorf("failed to get OptionsUrl %s for custom field %s", optionsUrl, field.Name)
} else {
options = fetchedOptions
}
}
}
customRegField := registrationField{
Name: "custom_fields." + field.Name,
DisplayedName: utils.SnakeCaseToHumanReadable(field.Name),
Type: registrationFieldType(field.Type),
Options: options,
Required: field.Required,
Description: field.Description,
OptionsUrl: optionsUrl,
}
regFields = append(regFields, customRegField)
}
return regFields
}
// Helper function to exclude pubkey field from marshaling into json
func excludePubKey(nss []*server_structs.Namespace) (nssNew []NamespaceWOPubkey) {
nssNew = make([]NamespaceWOPubkey, 0)
for _, ns := range nss {
nsNew := NamespaceWOPubkey{
ID: ns.ID,
Prefix: ns.Prefix,
Pubkey: ns.Pubkey,
AdminMetadata: ns.AdminMetadata,
Identity: ns.Identity,
}
nssNew = append(nssNew, nsNew)
}
return
}
func checkUniqueOptions(options []registrationFieldOption) bool {
repeatMap := make(map[string]bool)
for _, options := range options {
if repeatMap[options.ID] {
return false
} else {
repeatMap[options.ID] = true
}
}
return true
}
// Format custom registration fields in-place, by converting any float64/32 number to int
func formatCustomFields(customFields map[string]interface{}) {
for key, val := range customFields {
switch v := val.(type) {
case float64:
customFields[key] = int(v)
case float32:
customFields[key] = int(v)
}
}
}
// Initialize institutions list
func InitInstConfig(ctx context.Context, egrp *errgroup.Group) error {
institutions := []registrationFieldOption{}
if err := param.Registry_Institutions.Unmarshal(&institutions); err != nil {
log.Error("Fail to read Registry.Institutions. Make sure you had the correct format", err)
return errors.Wrap(err, "Fail to read Registry.Institutions. Make sure you had the correct format")
}
instRegIdx := -1 // From the registrationFields, find the index of the field admin_metadata.institution
for idx, reg := range registrationFields {
if reg.Name == "admin_metadata.institution" {
instRegIdx = idx
registrationFields[idx].Options = institutions
}
}
if instRegIdx == -1 {
return errors.New("fail to populate institution options. admin_metadata.institution does not exist in the list of registrationFields")
}
if param.Registry_InstitutionsUrl.GetString() != "" {
// Read from Registry.Institutions if Registry.InstitutionsUrl is empty
// or Registry.Institutions and Registry.InstitutionsUrl are both set
if len(institutions) > 0 {
log.Warning("Registry.Institutions and Registry.InstitutionsUrl are both set. Registry.InstitutionsUrl is ignored")
if !checkUniqueOptions(institutions) {
return errors.Errorf("Institution IDs read from config are not unique")
}
// return here so that we don't init the institution url cache
return nil
}
// Populate optionsUrl for institution field in registrationFields
registrationFields[instRegIdx].OptionsUrl = param.Registry_InstitutionsUrl.GetString()
instCacheTTL := param.Registry_InstitutionsUrlReloadMinutes.GetDuration()
institutions, err := getCachedOptions(param.Registry_InstitutionsUrl.GetString(), instCacheTTL)
if err != nil {
return err
}
registrationFields[instRegIdx].Options = institutions
}
if !checkUniqueOptions(institutions) {
return errors.Errorf("Institution IDs read from config are not unique")
}
// Else we will read from Registry.Institutions. No extra action needed.
return nil
}
// Initialize custom registration fields provided via Registry.CustomRegistrationFields
func InitCustomRegistrationFields() error {
configFields := []customRegFieldsConfig{}
if err := param.Registry_CustomRegistrationFields.Unmarshal(&configFields); err != nil {
return errors.Wrap(err, "Error reading from config value for Registry.CustomRegistrationFields")
}
customRegFieldsConfigs = configFields
fieldNames := make(map[string]bool, 0)
for _, conf := range configFields {
// Duplicated name check
if fieldNames[conf.Name] {
return errors.New(fmt.Sprintf("Bad custom registration fields, duplicated field name: %q", conf.Name))
} else {
fieldNames[conf.Name] = true
}
if conf.Type != "string" && conf.Type != "bool" && conf.Type != "int" && conf.Type != "enum" && conf.Type != "datetime" {
return errors.New(fmt.Sprintf("Bad custom registration field, unsupported field type: %q with %q", conf.Name, conf.Type))
}
if conf.Type == "enum" {
if (conf.Options == nil || len(conf.Options) == 0) && conf.OptionsUrl == "" {
return errors.New(fmt.Sprintf("Bad custom registration field, 'enum' type field does not have options or optionsUrl set: %q", conf.Name))
}
}
}
additionalRegFields := convertCustomRegFields(configFields)
registrationFields = append(registrationFields, additionalRegFields...)
return nil
}