-
Notifications
You must be signed in to change notification settings - Fork 4
/
validation.go
390 lines (337 loc) · 12.5 KB
/
validation.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
// Package htsrequest provides operations for parsing htsget-related
// parameters from the HTTP request, and performing validation and
// transformation
//
// Module validation defines functions for validating whether the value of
// HTTP request parameters are acceptable
package htsrequest
import (
"bufio"
"errors"
"net/http"
"os"
"os/exec"
"regexp"
"strings"
"github.com/ga4gh/htsget-refserver/internal/htsconfig"
"github.com/ga4gh/htsget-refserver/internal/htsconstants"
"github.com/ga4gh/htsget-refserver/internal/htserror"
"github.com/ga4gh/htsget-refserver/internal/htsutils"
"github.com/ga4gh/htsget-refserver/internal/awsutils"
)
// ParamValidator validates request parameters
type ParamValidator struct{}
// NewParamValidator instantiates a new ParamValidator object
func NewParamValidator() *ParamValidator {
return new(ParamValidator)
}
// errorsByParam (map[string]func(http.ResponseWriter, *string)): the correct
// error to raise for each request parameter validation
var errorsByParam = map[string]func(http.ResponseWriter, *string){
"id": htserror.NotFound,
"format": htserror.UnsupportedFormat,
"class": htserror.InvalidInput,
"referenceName": htserror.InvalidRange,
"start": htserror.InvalidRange,
"end": htserror.InvalidRange,
"fields": htserror.InvalidInput,
"tags": htserror.InvalidInput,
"notags": htserror.InvalidInput,
"regions": htserror.InvalidRange,
"HtsgetBlockClass": htserror.InvalidInput,
"HtsgetBlockId": htserror.InternalServerError,
"HtsgetNumBlocks": htserror.InternalServerError,
"HtsgetFilePath": htserror.InternalServerError,
"Range": htserror.InternalServerError,
}
// isGreaterThanEqualToZero determines if a string can be parsed as an integer,
// and is greater than or equal to zero
func isGreaterThanEqualToZero(num int) bool {
if num < 0 {
return false
}
return true
}
// NoValidation is an empty validation function for request parameters that do
// not need to be validated. always returns true
func (v *ParamValidator) NoValidation(htsgetReq *HtsgetRequest, value string) (bool, string) {
return true, ""
}
// ValidateID validates the 'id' parameter. checks if an object matching
// the 'id' could be found from the data source
func (v *ParamValidator) ValidateID(htsgetReq *HtsgetRequest, id string) (bool, string) {
objPath, err := htsconfig.GetObjectPath(htsgetReq.GetEndpoint(), id)
if err != nil {
return false, "The requested resource could not be associated with a registered data source"
}
// attempt to locate the object by http request (if url) or on local file
// path
if htsutils.IsValidURL(objPath) {
if strings.HasPrefix(objPath, awsutils.S3Proto) {
_, err := awsutils.HeadS3Object(awsutils.S3Dto{
ObjPath: objPath,
})
if err != nil {
return false, "Error accessing the requested S3 resource"
}
} else {
res, err := http.Head(objPath)
if err != nil {
return false, "The requested resource was not found"
}
res.Body.Close()
if res.Status == "404 Not Found" {
return false, "The requested resource was not found"
}
}
} else {
_, err := os.Stat(objPath)
if os.IsNotExist(err) {
return false, "The requested resource was not found"
}
}
return true, ""
}
// ValidateFormat validates the 'format' parameter. checks if the requested
// format is one of the allowed options based on endpoint
func (v *ParamValidator) ValidateFormat(htsgetReq *HtsgetRequest, format string) (bool, string) {
allowedFormats := htsgetReq.GetEndpoint().AllowedFormats()
if !htsutils.IsItemInArray(format, allowedFormats) {
return false, "file format: '" + format + "' not supported"
}
return true, ""
}
// ValidateClass validates the 'class' parameter. checks if the requested class
// is one of the allowed options
func (v *ParamValidator) ValidateClass(htsgetReq *HtsgetRequest, class string) (bool, string) {
switch strings.ToLower(class) {
case htsconstants.ClassHeader:
return true, ""
case htsconstants.ClassBody:
return false, "'body' only requests currently not supported" // currently not supported
default:
return false, "class: '" + class + "' not supported"
}
}
func getReferenceNamesInReadsObject(htsgetReq *HtsgetRequest) ([]string, error) {
var referenceNames []string
fileURL, err := htsconfig.GetObjectPath(htsgetReq.GetEndpoint(), htsgetReq.GetID())
if err != nil {
return nil, err
}
cmd := exec.Command("samtools", "view", "-H", fileURL)
pipe, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
cmd.Start()
reader := bufio.NewReader(pipe)
l, _, err := reader.ReadLine()
for ; err == nil; l, _, err = reader.ReadLine() {
pattern := regexp.MustCompile("^@SQ\tSN:(.+?)\t.+?$")
submatches := pattern.FindStringSubmatch(string(l))
if len(submatches) > 1 {
referenceNames = append(referenceNames, submatches[1])
}
}
// wait till job completes, if the exit code wasn't 0, then raise an error
// likely the file was not found
err = cmd.Wait()
if err != nil {
return nil, errors.New("Could not get referenceNames from requested alignment file")
}
return referenceNames, nil
}
func getReferenceNamesInVariantsObject(htsgetReq *HtsgetRequest) ([]string, error) {
var referenceNames []string
fileURL, err := htsconfig.GetObjectPath(htsgetReq.GetEndpoint(), htsgetReq.GetID())
if err != nil {
return nil, err
}
cmd := exec.Command("bcftools", "view", "-h", fileURL)
pipe, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
cmd.Start()
reader := bufio.NewReader(pipe)
l, _, err := reader.ReadLine()
for ; err == nil; l, _, err = reader.ReadLine() {
pattern := regexp.MustCompile("^##contig=<.*?ID=(.+?)[,>]")
submatches := pattern.FindStringSubmatch(string(l))
if len(submatches) > 1 {
referenceNames = append(referenceNames, submatches[1])
}
}
// wait till job completes, if the exit code wasn't 0, then raise an error
// likely the file was not found
err = cmd.Wait()
if err != nil {
return nil, errors.New("Could not get referenceNames from requested variant file")
}
return referenceNames, nil
}
// getAllowedReferenceNames
// for a given endpoint (BAM request / VCF request), return the allowable values
// for the 'referenceName' parameter for the requested object
func getReferenceNames(htsgetReq *HtsgetRequest) ([]string, error) {
functions := map[htsconstants.APIEndpoint]func(htsgetReq *HtsgetRequest) ([]string, error){
htsconstants.APIEndpointReadsTicket: getReferenceNamesInReadsObject,
htsconstants.APIEndpointReadsData: getReferenceNamesInReadsObject,
htsconstants.APIEndpointVariantsTicket: getReferenceNamesInVariantsObject,
htsconstants.APIEndpointVariantsData: getReferenceNamesInVariantsObject,
}
return functions[htsgetReq.endpoint](htsgetReq)
}
// ValidateReferenceName validates the 'referenceName' query string
// parameter. checks if the requested reference contig/chromosome is in the
// BAM/CRAM header sequence dictionary. if unplaced unmapped reads are requested
// (*), do not perform validation
func (v *ParamValidator) ValidateReferenceName(htsgetReq *HtsgetRequest, referenceName string) (bool, string) {
// incompatible with header only request
if htsgetReq.HeaderOnlyRequested() {
return false, "'referenceName' incompatible with header-only request"
}
// no validation if '*' was requested
if referenceName == "*" {
return true, ""
}
// otherwise, check that referenceName is in the header
referenceNames, err := getReferenceNames(htsgetReq)
if err != nil {
return false, err.Error()
}
if htsutils.IsItemInArray(referenceName, referenceNames) {
return true, ""
}
return false, "invalid 'referenceName': " + referenceName
}
// ValidateStart validates the 'start' query string parameter. checks that it is
// a valid, non-zero integer, and that it is being used correctly in conjunction
// with 'referenceName'
func (v *ParamValidator) ValidateStart(htsgetReq *HtsgetRequest, start int) (bool, string) {
// incompatible with header only request
if htsgetReq.HeaderOnlyRequested() {
return false, "'start' incompatible with header-only request"
}
// start requires referenceName to specify a true chromosome
if htsgetReq.UnplacedUnmappedReadsRequested() {
return false, "'start' cannot be requested with unplaced, unmapped reads"
}
// start requires referenceName to be specified as well
if !htsgetReq.ReferenceNameRequested() {
return false, "'start' cannot be set without 'referenceName'"
}
// start must be >= 0
if !isGreaterThanEqualToZero(start) {
return false, "'start' must be greater than or equal to zero"
}
return true, ""
}
// ValidateEnd validates the 'end' query string parameter. checks that it is a
// valid, non-zero integer, that it's being used correctly in conjunction with
// 'referenceName', and that the end coordinate is greater than the start
// coordinate
func (v *ParamValidator) ValidateEnd(htsgetReq *HtsgetRequest, end int) (bool, string) {
// incompatible with header only request
if htsgetReq.HeaderOnlyRequested() {
return false, "'end' incompatible with header-only request"
}
start := htsgetReq.GetStart()
// end requires referenceName to specify a true chromosome
if htsgetReq.UnplacedUnmappedReadsRequested() {
return false, "'end' cannot be requested with unplaced, unmapped reads"
}
// end requires referenceName to be specified as well
if !htsgetReq.ReferenceNameRequested() {
return false, "'end' cannot be set without 'referenceName'"
}
// end must be >= 0
if !isGreaterThanEqualToZero(end) {
return false, "'end' must be greater than or equal to zero"
}
// if start is specified, end must be greater than start
if start != -1 {
if start >= end {
return false, "'end' MUST be higher than 'start'"
}
}
return true, ""
}
// ValidateFields validates 'fields' parameter. every requested field must
// have an acceptable BAM/CRAM column name
func (v *ParamValidator) ValidateFields(htsgetReq *HtsgetRequest, fields []string) (bool, string) {
// incompatible with header only request
if htsgetReq.HeaderOnlyRequested() {
return false, "'fields' incompatible with header-only request"
}
for _, fieldItem := range fields {
if _, ok := htsconstants.BamFields[fieldItem]; !ok {
return false, "'" + fieldItem + "' not an acceptable field"
}
}
return true, ""
}
// ValidateTags only validates that tags hasn't been requested alongside
// a header only request
func (v *ParamValidator) ValidateTags(htsgetReq *HtsgetRequest, tags []string) (bool, string) {
if htsgetReq.HeaderOnlyRequested() {
return false, "'tags' incompatible with header-only request"
}
return true, ""
}
// ValidateNoTags validates the 'notags' query string parameter. checks that
// there is no overlap between tags included by 'tags' and tags excluded by
// 'notags'
func (v *ParamValidator) ValidateNoTags(htsgetReq *HtsgetRequest, notags []string) (bool, string) {
// incompatible with header only request
if htsgetReq.HeaderOnlyRequested() {
return false, "'notags' incompatible with header-only request"
}
tags := htsgetReq.GetTags()
for _, tagItem := range tags {
for _, notagItem := range notags {
if tagItem == notagItem {
return false, "'" + tagItem + "' cannot be in both 'tags' and 'notags'"
}
}
}
return true, ""
}
// ValidateRegions validates whether every region within an array of regions is
// valid, that is, contains acceptable referenceName, start, and end values
func (v *ParamValidator) ValidateRegions(htsgetReq *HtsgetRequest, regions []*Region) (bool, string) {
allowedReferenceNames, err := getReferenceNames(htsgetReq)
if err != nil {
return false, err.Error()
}
for _, region := range regions {
if region.ReferenceNameRequested() {
if !htsutils.IsItemInArray(region.GetReferenceName(), allowedReferenceNames) {
return false, "Invalid referenceName in regions list: '" + region.GetReferenceName() + "'"
}
}
if region.StartRequested() {
if !region.ReferenceNameRequested() {
return false, "Invalid region(s): 'start' cannot be set without 'referenceName'"
}
if !isGreaterThanEqualToZero(region.GetStart()) {
return false, "Invalid region(s): 'start' MUST be greater than or equal to zero"
}
}
if region.EndRequested() {
if !region.ReferenceNameRequested() {
return false, "Invalid region(s): 'end' cannot be set without 'referenceName'"
}
if !isGreaterThanEqualToZero(region.GetEnd()) {
return false, "Invalid region(s): 'end' MUST be greater than or equal to zero"
}
if region.StartRequested() {
if region.GetStart() >= region.GetEnd() {
return false, "Invalid region(s): 'end' MUST be greater than 'start'"
}
}
}
}
return true, ""
}