/
testing.py
145 lines (119 loc) · 5.04 KB
/
testing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
import logging
from pathlib import Path
from typing import List, Optional
import yaml
from nomenclature.definition import DataStructureDefinition
from nomenclature.processor import RegionProcessor, RequiredDataValidator
from nomenclature.error import ErrorCollector
logger = logging.getLogger(__name__)
ILLEGAL_CHARS = ["\u202f"]
def assert_valid_yaml(path: Path):
"""Assert that all yaml files in `path` can be parsed without errors"""
special_characters = ""
# iterate over the yaml files in all sub-folders and try loading each
error = False
for file in (f for f in path.glob("**/*") if f.suffix in {".yaml", ".yml"}):
try:
with open(file, "r", encoding="utf-8") as stream:
yaml.safe_load(stream)
except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
error = True
logger.error(f"Error parsing file {e}")
with open(file, "r", encoding="utf-8") as all_lines:
# check if any special character is found in the file
for index, line in enumerate(all_lines.readlines()):
for col, char in enumerate(line):
if char in ILLEGAL_CHARS:
special_characters += (
f"\n - {file.name}, line {index + 1}, col {col + 1}. "
)
if special_characters:
raise AssertionError(f"Unexpected special character(s): {special_characters}")
# test fails if any file cannot be parsed, raise error with list of these files
if error:
raise AssertionError(
"Parsing the yaml files failed. Please check the log for details."
)
def _check_mappings(
path: Path,
definitions: str = "definitions",
dimensions: Optional[List[str]] = None,
mappings: Optional[str] = None,
) -> None:
dsd = DataStructureDefinition(path / definitions, dimensions)
if mappings is None:
if (path / "mappings").is_dir():
RegionProcessor.from_directory(path / "mappings", dsd)
elif (path / mappings).is_dir():
RegionProcessor.from_directory(path / mappings, dsd)
else:
raise FileNotFoundError(f"Mappings directory not found: {path / mappings}")
def _collect_RequiredData_errors(
required_data_dir: Path, dsd: DataStructureDefinition
) -> None:
errors = ErrorCollector()
for file in required_data_dir.iterdir():
try:
RequiredDataValidator.from_file(file).validate_with_definition(dsd)
except ValueError as error:
errors.append(error)
if errors:
raise ValueError(f"Found error(s) in required data files:\n{errors}")
def _check_RequiredData(
path: Path,
definitions: str = "definitions",
dimensions: Optional[List[str]] = None,
required_data: Optional[str] = None,
) -> None:
dsd = DataStructureDefinition(path / definitions, dimensions)
if required_data is None:
if (path / "requiredData").is_dir():
_collect_RequiredData_errors(path / "required_data", dsd)
elif (path / required_data).is_dir():
_collect_RequiredData_errors(path / required_data, dsd)
else:
raise FileNotFoundError(
f"Directory for required data not found at: {path / required_data}"
)
def assert_valid_structure(
path: Path,
definitions: str = "definitions",
mappings: Optional[str] = None,
required_data: Optional[str] = None,
dimensions: Optional[List[str]] = None,
) -> None:
"""Assert that `path` can be initialized as a :class:`DataStructureDefinition`
Parameters
----------
path : Path
Project directory to be validated
definitions : str, optional
Name of the definitions folder, defaults to "definitions"
mappings : str, optional
Name of the mappings folder, defaults to "mappings" (if this folder exists)
required_data : str, optional
Name of the required data folder, defaults to "required_data" (if this folder
exists)
dimensions : List[str], optional
Dimensions to be checked, defaults to all sub-folders of `definitions`
Notes
-----
Folder structure of `path`:
- A `definitions` folder is required and must be a valid
:class:`DataStructureDefinition`
- The `definitions` folder must contain sub-folder(s) to validate
- If a `mappings` folder exists, it must be a valid :class:`RegionProcessor`
"""
if not (path / definitions).is_dir():
raise NotADirectoryError(
f"Definitions directory not found: {path / definitions}"
)
if dimensions == (): # if "dimensions" were not specified
dimensions = [x.stem for x in (path / definitions).iterdir() if x.is_dir()]
if not dimensions:
raise FileNotFoundError(
f"`definitions` directory is empty: {path / definitions}"
)
_check_mappings(path, definitions, dimensions, mappings)
_check_RequiredData(path, definitions, dimensions, required_data)
# Todo: add function which runs `DataStructureDefinition(path).validate(scenario)`