Skip to content

Commit

Permalink
schema and yaml: correct bugs in schema and use safe YAML load/dump
Browse files Browse the repository at this point in the history 
* schema: rename some "id" values, and fix a major bug where the value of "items" was given as an array of one JSON schema, thereby only validating the first element of a list instead of *all* elements in the list.
* testsuite: fix test data to match newly corrected schema.
* yaml: always use SafeLoader and SafeDumper or preferably the corresponding LibYAML classes if installed.
* yaml: simplify validation of parser errors by catching a generic Exception rather than yaml.parser.ParserError and IOError, also now implicitly catching a yaml.constructor.ConstructorError caused by the presence of "!!python/unicode" tags.
* version: bump to 0.1.16.

Signed-off-by: Graeme Watt <graeme.watt@durham.ac.uk>
  • Loading branch information
@GraemeWatt
GraemeWatt committed Jun 29, 2017
1 parent 1c74478 commit fe01816
Show file tree
Hide file tree
Showing 7 changed files with 230 additions and 267 deletions.
43 changes: 11 additions & 32 deletions hepdata_validator/data_file_validator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,12 @@

import os
import yaml
from yaml.scanner import ScannerError
from yaml.parser import ParserError

# We try to load using the CSafeLoader for speed improvements.
try:
from yaml import CSafeLoader as Loader
except ImportError: #pragma: no cover
from yaml import SafeLoader as Loader #pragma: no cover

from hepdata_validator import Validator, ValidationMessage
from jsonschema import validate as json_validate, ValidationError
Expand Down Expand Up @@ -89,36 +93,11 @@ def validate(self, **kwargs):
if data is None:

try:
# We try to load using the CLoader for speed improvements.
try:
data = yaml.load(open(file_path, 'r'), Loader=yaml.CLoader)
except ScannerError as se:
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + str(se)))
return False
except ParserError as pe:
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + pe.__str__()))
return False
except IOError as ioe:
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + ioe.__str__()))
return False
except: #pragma: no cover
try: # pragma: no cover
data = yaml.load(open(file_path, 'r')) # pragma: no cover
except ScannerError as se: # pragma: no cover
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + str(se))) # pragma: no cover
return False
except ParserError as pe: # pragma: no cover
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + pe.__str__()))
return False
except IOError as ioe:
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + ioe.__str__()))
return False
data = yaml.load(open(file_path, 'r'), Loader=Loader)
except Exception as e:
self.add_validation_message(ValidationMessage(file=file_path, message=
'There was a problem parsing the file.\n' + e.__str__()))
return False

try:

Expand Down
224 changes: 108 additions & 116 deletions hepdata_validator/schemas/additional_info_schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,30 @@
"record_ids": {
"id": "http://jsonschema.net/record_ids",
"type": "array",
"items": [
{
"id": "http://jsonschema.net/record_ids/record_id",
"type": "object",
"properties": {
"type": {
"id": "http://jsonschema.net/record_ids/record_id/type",
"type": [
"string"
]
},
"id": {
"id": "http://jsonschema.net/record_ids/record_id/id",
"type": [
"string",
"number"
]
}
"items": {
"id": "http://jsonschema.net/record_ids/record_id",
"type": "object",
"properties": {
"type": {
"id": "http://jsonschema.net/record_ids/record_id/type",
"type": [
"string"
]
},
"additionalProperties": false,
"required": [
"type",
"id"
]
}
]
"id": {
"id": "http://jsonschema.net/record_ids/record_id/id",
"type": [
"string",
"number"
]
}
},
"additionalProperties": false,
"required": [
"type",
"id"
]
}
},
"preprintyear": {
"id": "http://hepdata.org/submission/schema/data/preprintyear",
Expand All @@ -60,109 +58,103 @@
"modifications": {
"id": "http://jsonschema.net/modifications",
"type": "array",
"items": [
{
"id": "http://jsonschema.net/modifications/modification",
"type": "object",
"properties": {
"action": {
"id": "http://jsonschema.net/modifications/modification/action",
"type": "string"
},
"who": {
"id": "http://jsonschema.net/modifications/modification/who",
"type": "string"
}
"items": {
"id": "http://jsonschema.net/modifications/modification",
"type": "object",
"properties": {
"action": {
"id": "http://jsonschema.net/modifications/modification/action",
"type": "string"
},
"additionalProperties": true,
"required": [
"action",
"who"
]
}
]
"who": {
"id": "http://jsonschema.net/modifications/modification/who",
"type": "string"
}
},
"additionalProperties": true,
"required": [
"action",
"who"
]
}
},
"additional_resources": {
"id": "http://jsonschema.net/additional_resources",
"type": "array",
"items": [
{
"id": "http://jsonschema.net/additional_resources/resource",
"type": "object",
"properties": {
"location": {
"id": "http://jsonschema.net/additional_resources/resource/location",
"type": "string"
},
"description": {
"id": "http://jsonschema.net/additional_resources/resource/description",
"type": "string"
},
"license": {
"id": "http://jsonschema.net/additional_resources/resource/license",
"type": "object",
"properties": {
"name": {
"id": "http://jsonschema.net/additional_resources/resource/license/name",
"type": "string"
},
"url": {
"id": "http://jsonschema.net/additional_resources/resource/license/url",
"type": "string"
},
"description": {
"id": "http://jsonschema.net/additional_resources/resource/license/description",
"type": "string"
}
},
"additionalProperties": false,
"required": [
"name",
"url"
]
}
"items": {
"id": "http://jsonschema.net/additional_resources/resource",
"type": "object",
"properties": {
"location": {
"id": "http://jsonschema.net/additional_resources/resource/location",
"type": "string"
},
"description": {
"id": "http://jsonschema.net/additional_resources/resource/description",
"type": "string"
},
"required": [
"location"
]
}
]
"license": {
"id": "http://jsonschema.net/additional_resources/resource/license",
"type": "object",
"properties": {
"name": {
"id": "http://jsonschema.net/additional_resources/resource/license/name",
"type": "string"
},
"url": {
"id": "http://jsonschema.net/additional_resources/resource/license/url",
"type": "string"
},
"description": {
"id": "http://jsonschema.net/additional_resources/resource/license/description",
"type": "string"
}
},
"additionalProperties": false,
"required": [
"name",
"url"
]
}
},
"required": [
"location"
]
}
},
"associated_records": {
"id": "http://jsonschema.net/associated_records",
"type": "array",
"description": "Links to other HEPData Submissions or INSPIRE Records that relate to this submission.",
"items": [
{
"id": "http://jsonschema.net/associated_records/output",
"type": "object",
"properties": {
"type": {
"id": "http://jsonschema.net/associated_records/output/type",
"type": "string"
},
"identifier": {
"id": "http://jsonschema.net/associated_records/output/identifier",
"type": [
"string",
"number"
]
},
"description": {
"id": "http://jsonschema.net/associated_records/output/description",
"type": "string"
},
"url": {
"id": "http://jsonschema.net/associated_records/output/url",
"type": "string"
}
"items": {
"id": "http://jsonschema.net/associated_records/output",
"type": "object",
"properties": {
"type": {
"id": "http://jsonschema.net/associated_records/output/type",
"type": "string"
},
"identifier": {
"id": "http://jsonschema.net/associated_records/output/identifier",
"type": [
"string",
"number"
]
},
"description": {
"id": "http://jsonschema.net/associated_records/output/description",
"type": "string"
},
"required": [
"identifier",
"type"
]
}
]
"url": {
"id": "http://jsonschema.net/associated_records/output/url",
"type": "string"
}
},
"required": [
"identifier",
"type"
]
}
},
"comment": {
"id": "http://hepdata.org/submission/schema/additional_info/comment",
Expand Down
4 changes: 2 additions & 2 deletions hepdata_validator/schemas/data_schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@
]
},
"low": {
"id": "http://hepdata.org/submission/schema/data/independent_variables/0/values/1/value",
"id": "http://hepdata.org/submission/schema/data/independent_variables/0/values/1/low",
"type": "number"
},
"high": {
"id": "http://hepdata.org/submission/schema/data/independent_variables/0/values/1/value",
"id": "http://hepdata.org/submission/schema/data/independent_variables/0/values/1/high",
"type": "number"
}
},
Expand Down

0 comments on commit fe01816

Please sign in to comment.