Skip to content

Commit aa6feaf

Browse files
committed
Merge branch 'develop' of ssh://git.iter.org/imas/imaspy into feature/simpler-env
2 parents ebef667 + b5c8759 commit aa6feaf

22 files changed

+626
-111
lines changed

conftest.py

+5
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,11 @@ def pytest_generate_tests(metafunc):
7171
if name.endswith("_bool"):
7272
metafunc.parametrize(name, [True, False])
7373

74+
@pytest.fixture
75+
def fake_toplevel_xml():
76+
from pathlib import Path
77+
78+
return Path(__file__).parent / "imaspy/assets/IDS_fake_toplevel.xml"
7479

7580
@pytest.fixture
7681
def ids_minimal():

docs/source/conf.py

+1
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@
121121
"sphinx_rtd_theme", # Theme
122122
"recommonmark", # For markdown support, does not support 'full' CommonMark syntax (yet)!
123123
"sphinx_autodoc_typehints", # Auto-parse type hints. Napoleon BEFORE typehints
124+
"sphinxcontrib.mermaid", # Draw graphs using Mermaid.js
124125
]
125126

126127
# Add any paths that contain templates here, relative to this directory.

docs/source/dd_load_callgraph.rst

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
.. mermaid::
2+
3+
flowchart TD
4+
A[shot, run] --> root
5+
R[Reference shot\nReference run] --> root
6+
V[Version in memory e.g. 3.38.1] --> VOX
7+
X[XML path e.g. IDSRoot.xml] --> VOX
8+
VD[Version on disk memory e.g. 3.37.1] --> VOXD
9+
XD[XML path on disk memory e.g. 3.37.1] --> VOXD
10+
VOX{Version or XML\nin memory?} --> root
11+
VOXD{Version or XML\non disk?} --> root
12+
L[lazy] --> root
13+
root[IDSRoot] --> LZ{Lazy?}
14+
LZ -->|True| D[Loop over direct children with a name\nInitialize IDSToplevel with it]
15+
N[version is handled special here!] --> D
16+
LZ-->|False| E[Add name to children, nothing else]
17+
E -->|on getattr| F{attr style?}
18+
F -->|already initialized| G["return using super()"]
19+
F -->|No children| H["return []"]
20+
F -->|key in _children| I["Initialize IDSToplevel with key"]
21+
I --> T
22+
D -->|parent, key/name, ids XML element, backend_version, backend_xml_path| T[IDSToplevel init]
23+
T -->|"call super()"|K[Got parent, name, structure_xml]
24+
subgraph STRUCT["IDSStructure.__init__"]
25+
direction TB
26+
K --> B
27+
subgraph MIXIN["IDSMixin.__init__"]
28+
direction TB
29+
B[Got parent, name, structure_xml] --> J["Set internal vars on self:\n _name, _parent, _coordinates, _last_backend_xml_hash, _backend_name"]
30+
end
31+
J -->|For each XML child| M[Append name to _children]
32+
M --> O{child data_type}
33+
O -->|structure| P[Initialize childs IDSStructure]
34+
P -->|recurse, parent->child| K
35+
O -->|struct_array| Q[Initialize IDSStructArray]
36+
O -->|else| S["get_coordinates (can be {})"]
37+
S --> U["create_leaf_container(child name, child data_type, parent, coordinates, var_type=type)"]
38+
subgraph LEAF["create_leaf_container"]
39+
U --> W{Depending on\ndata_type}
40+
W --> Y[Determine ids_type, ndims from DD_TYPES]
41+
Y --> Z{ndins ==?}
42+
Z -->|ndins==0| AA["IDSPrimitive(name, ids_type, ndims, **kwargs)"]
43+
Z -->|ndins!=0 && ids_type==STR| AB["IDSPrimitive(name, ids_type, ndims, **kwargs)"]
44+
Z -->|ndins!=0 && ids_type!=STR| AC["IDSNumericArray(name, ids_type, ndims, **kwargs)"]
45+
end
46+
subgraph NUMERICARRAY["IDSNumericArray.__init__"]
47+
direction TB
48+
AC --> AD["No special init"]
49+
50+
end
51+
subgraph PRIMITIVE["IDSPrimitive.__init__"]
52+
direction TB
53+
AA --> AE
54+
AD --> AE
55+
AB --> AE["Same call signature, same case?"]
56+
AE -->|"Call super() IDSMixin init"| AF
57+
subgraph CHILDMIXIN["IDSMixin.__init__"]
58+
direction TB
59+
AF[Got parent, name, coordinates] --> AG["Set internal vars on self:\n _name, _parent, _coordinates, _last_backend_xml_hash, _backend_name"]
60+
end
61+
AG --> AH["Set internal vars on self:\n __value, _ids_type, _var_type, _ndims, _backend_type=None, _backend_ndims = None"]
62+
AH --> AI(["A child node has been generated\nPass to parent"])
63+
end
64+
end

envs/common/70_pytest_imaspy.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ if [ "$RUNMODE" == "normal" ] || [ "$RUNMODE" == "env" ]; then
2727
export COV_FLAGS=${COV_FLAGS:-'--cov=imaspy --cov-report=term --cov-report=xml:./coverage.xml'}
2828
export JUNIT_FLAGS=${JUNIT_FLAGS:-'--junit-xml=./junit.xml'}
2929
export PYTEST_MARK=${PYTEST_MARK:-''}
30+
export PYTEST_FILE_OR_DIR=${PYTEST_FILE_OR_DIR:-'../imaspy'}
3031
export IDSS=${IDSS:-pulse_schedule,ece}
3132

3233
source $IMASPY_VENV/bin/activate
@@ -46,7 +47,7 @@ if [ "$RUNMODE" == "normal" ] || [ "$RUNMODE" == "only" ]; then
4647
if [ $TESTSET == "mini" ]; then
4748
# Do not exit when tests fail
4849
set +e
49-
$PYTEST --ids=$IDSS $PYTEST_FLAGS $COV_FLAGS $JUNIT_FLAGS -m "$PYTEST_MARK" ../imaspy
50+
$PYTEST --ids=$IDSS $PYTEST_FLAGS $COV_FLAGS $JUNIT_FLAGS -m "$PYTEST_MARK" "$PYTEST_FILE_OR_DIR"
5051
set -e
5152
else
5253
echo Untested! Dropping shell!

imaspy/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# This file is part of IMASPy.
22
# You should have received IMASPy LICENSE file with this project.
33

4-
from distutils.version import StrictVersion as V
4+
from packaging.version import Version as V
55

66
import pkg_resources
77

imaspy/assets/IDS_fake_toplevel.xml

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<IDSs xmlns:fn="http://www.w3.org/2005/02/xpath-functions">
3+
<version>0.0.1</version>
4+
<cocos>11</cocos>
5+
<!-- A dressed down copy of Gyrokintec IDS 3.38.0 -->
6+
<utilities>
7+
<!-- Utilities copied at an as-needed basis -->
8+
</utilities>
9+
<IDS name="gyrokinetics"
10+
maxoccur="1"
11+
documentation="Description of a gyrokinetic simulation (delta-f, flux-tube). All quantities within this IDS are normalised (apart from time and from the normalizing quantities structure), thus independent of rhostar, consistently with the local approximation and a spectral representation is assumed in the perpendicular plane (i.e. homogeneous turbulence)."
12+
lifecycle_status="alpha"
13+
lifecycle_version="3.22.0"
14+
lifecycle_last_change="3.37.0"
15+
url="https://gitlab.com/gkdb/gkdb/raw/master/doc/general/IOGKDB.pdf">
16+
<field name="ids_properties"
17+
structure_reference="ids_properties"
18+
path="ids_properties"
19+
path_doc="ids_properties"
20+
documentation="Interface Data Structure properties. This element identifies the node above as an IDS"
21+
data_type="structure">
22+
<field name="comment"
23+
path="ids_properties/comment"
24+
path_doc="ids_properties/comment"
25+
documentation="Any comment describing the content of this IDS"
26+
data_type="STR_0D"
27+
type="constant"/>
28+
<field name="homogeneous_time"
29+
path="ids_properties/homogeneous_time"
30+
path_doc="ids_properties/homogeneous_time"
31+
documentation="This node must be filled (with 0, 1, or 2) for the IDS to be valid. If 1, the time of this IDS is homogeneous, i.e. the time values for this IDS are stored in the time node just below the root of this IDS. If 0, the time values are stored in the various time fields at lower levels in the tree. In the case only constant or static nodes are filled within the IDS, homogeneous_time must be set to 2"
32+
data_type="INT_0D"
33+
type="constant"/>
34+
<field name="source"
35+
path="ids_properties/source"
36+
path_doc="ids_properties/source"
37+
documentation="Source of the data (any comment describing the origin of the data : code, path to diagnostic signals, processing method, ...). Superseeded by the new provenance structure."
38+
data_type="STR_0D"
39+
type="constant"
40+
lifecycle_status="obsolescent"
41+
lifecycle_version="3.34.0"/>
42+
<field name="provider"
43+
path="ids_properties/provider"
44+
path_doc="ids_properties/provider"
45+
documentation="Name of the person in charge of producing this data"
46+
data_type="STR_0D"
47+
type="constant"/>
48+
<field name="creation_date"
49+
path="ids_properties/creation_date"
50+
path_doc="ids_properties/creation_date"
51+
documentation="Date at which this data has been produced"
52+
data_type="STR_0D"
53+
type="constant"/>
54+
<field name="version_put"
55+
structure_reference="version_dd_al"
56+
path="ids_properties/version_put"
57+
documentation="Version of the access layer package used to PUT this IDS"
58+
data_type="structure"
59+
path_doc="ids_properties/version_put">
60+
<field name="data_dictionary"
61+
path="ids_properties/version_put/data_dictionary"
62+
path_doc="ids_properties/version_put/data_dictionary"
63+
documentation="Version of Data Dictionary used to PUT this IDS"
64+
data_type="STR_0D"
65+
type="constant"/>
66+
<field name="access_layer"
67+
path="ids_properties/version_put/access_layer"
68+
path_doc="ids_properties/version_put/access_layer"
69+
documentation="Version of Access Layer used to PUT this IDS"
70+
data_type="STR_0D"
71+
type="constant"/>
72+
<field name="access_layer_language"
73+
path="ids_properties/version_put/access_layer_language"
74+
path_doc="ids_properties/version_put/access_layer_language"
75+
documentation="Programming language of the Access Layer high level API used to PUT this IDS"
76+
data_type="STR_0D"
77+
type="constant"/>
78+
</field>
79+
<field name="provenance"
80+
structure_reference="ids_provenance"
81+
path="ids_properties/provenance"
82+
documentation="Provenance information about this IDS"
83+
data_type="structure"
84+
path_doc="ids_properties/provenance"
85+
lifecycle_status="alpha"
86+
lifecycle_version="3.34.0">
87+
<field name="node"
88+
structure_reference="ids_provenance_node"
89+
path="ids_properties/provenance/node"
90+
documentation="Set of IDS nodes for which the provenance is given. The provenance information applies to the whole structure below the IDS node. For documenting provenance information for the whole IDS, set the size of this array of structure to 1 and leave the child &#34;path&#34; node empty"
91+
data_type="struct_array"
92+
maxoccur="20"
93+
path_doc="ids_properties/provenance/node(i1)"
94+
type="constant"
95+
coordinate1="1...N">
96+
<field name="path"
97+
path="ids_properties/provenance/node/path"
98+
path_doc="ids_properties/provenance/node(i1)/path"
99+
documentation="Path of the node within the IDS, following the syntax given in the link below. If empty, means the provenance information applies to the whole IDS."
100+
data_type="STR_0D"
101+
type="constant"
102+
url="utilities/IDS-path-syntax.md"/>
103+
<field name="sources"
104+
path="ids_properties/provenance/node/sources"
105+
path_doc="ids_properties/provenance/node(i1)/sources(:)"
106+
documentation="List of sources used to import or calculate this node, identified as explained below. In case the node is the result of of a calculation / data processing, the source is an input to the process described in the &#34;code&#34; structure at the root of the IDS. The source can be an IDS (identified by a URI or a persitent identifier, see syntax in the link below) or non-IDS data imported directly from an non-IMAS database (identified by the command used to import the source, or the persistent identifier of the data source). Often data are obtained by a chain of processes, however only the last process input are recorded here. The full chain of provenance has then to be reconstructed recursively from the provenance information contained in the data sources."
107+
data_type="STR_1D"
108+
type="constant"
109+
coordinate1="1...N"
110+
url="utilities/IMAS-URI-scheme.md"/>
111+
</field>
112+
</field>
113+
</field>
114+
<field name="wavevector"
115+
structure_reference="gyrokinetics_wavevector"
116+
path="wavevector"
117+
documentation="Set of wavevectors"
118+
data_type="struct_array"
119+
maxoccur="10"
120+
path_doc="wavevector(i1)"
121+
coordinate1="1...N">
122+
<field name="radial_component_norm"
123+
path="wavevector/radial_component_norm"
124+
path_doc="wavevector(i1)/radial_component_norm"
125+
documentation="Normalised radial component of the wavevector"
126+
data_type="FLT_0D"
127+
type="constant"
128+
units="-"/>
129+
<field name="radial_component_norm_error_upper"
130+
path="wavevector/radial_component_norm_error_upper"
131+
path_doc="wavevector(i1)/radial_component_norm_error_upper"
132+
documentation="Upper error for &#34;radial_component_norm&#34;"
133+
data_type="FLT_0D"
134+
type="constant"
135+
units="-"/>
136+
<field name="radial_component_norm_error_lower"
137+
path="wavevector/radial_component_norm_error_lower"
138+
path_doc="wavevector(i1)/radial_component_norm_error_lower"
139+
documentation="Lower error for &#34;radial_component_norm&#34;"
140+
data_type="FLT_0D"
141+
type="constant"
142+
units="-"/>
143+
<field name="radial_component_norm_error_index"
144+
path="wavevector/radial_component_norm_error_index"
145+
path_doc="wavevector(i1)/radial_component_norm_error_index"
146+
documentation="Index in the error_description list for &#34;radial_component_norm&#34;"
147+
data_type="int_type"
148+
type="constant"/>
149+
<field name="eigenmode"
150+
structure_reference="gyrokinetics_eigenmode"
151+
path="wavevector/eigenmode"
152+
documentation="Set of eigenmode for this wavector"
153+
data_type="struct_array"
154+
maxoccur="5"
155+
path_doc="wavevector(i1)/eigenmode(i2)"
156+
coordinate1="1...N">
157+
<field name="growth_rate_norm"
158+
path="wavevector/eigenmode/growth_rate_norm"
159+
path_doc="wavevector(i1)/eigenmode(i2)/growth_rate_norm"
160+
documentation="Growth rate"
161+
data_type="FLT_0D"
162+
type="constant"
163+
units="-"/>
164+
<field name="growth_rate_norm_error_upper"
165+
path="wavevector/eigenmode/growth_rate_norm_error_upper"
166+
path_doc="wavevector(i1)/eigenmode(i2)/growth_rate_norm_error_upper"
167+
documentation="Upper error for &#34;growth_rate_norm&#34;"
168+
data_type="FLT_0D"
169+
type="constant"
170+
units="-"/>
171+
<field name="growth_rate_norm_error_lower"
172+
path="wavevector/eigenmode/growth_rate_norm_error_lower"
173+
path_doc="wavevector(i1)/eigenmode(i2)/growth_rate_norm_error_lower"
174+
documentation="Lower error for &#34;growth_rate_norm&#34;"
175+
data_type="FLT_0D"
176+
type="constant"
177+
units="-"/>
178+
<field name="growth_rate_norm_error_index"
179+
path="wavevector/eigenmode/growth_rate_norm_error_index"
180+
path_doc="wavevector(i1)/eigenmode(i2)/growth_rate_norm_error_index"
181+
documentation="Index in the error_description list for &#34;growth_rate_norm&#34;"
182+
data_type="int_type"
183+
type="constant"/>
184+
<field name="frequency_norm"
185+
path="wavevector/eigenmode/frequency_norm"
186+
path_doc="wavevector(i1)/eigenmode(i2)/frequency_norm"
187+
documentation="Frequency"
188+
data_type="FLT_0D"
189+
type="constant"
190+
units="-"/>
191+
<field name="frequency_norm_error_upper"
192+
path="wavevector/eigenmode/frequency_norm_error_upper"
193+
path_doc="wavevector(i1)/eigenmode(i2)/frequency_norm_error_upper"
194+
documentation="Upper error for &#34;frequency_norm&#34;"
195+
data_type="FLT_0D"
196+
type="constant"
197+
units="-"/>
198+
<field name="frequency_norm_error_lower"
199+
path="wavevector/eigenmode/frequency_norm_error_lower"
200+
path_doc="wavevector(i1)/eigenmode(i2)/frequency_norm_error_lower"
201+
documentation="Lower error for &#34;frequency_norm&#34;"
202+
data_type="FLT_0D"
203+
type="constant"
204+
units="-"/>
205+
<field name="frequency_norm_error_index"
206+
path="wavevector/eigenmode/frequency_norm_error_index"
207+
path_doc="wavevector(i1)/eigenmode(i2)/frequency_norm_error_index"
208+
documentation="Index in the error_description list for &#34;frequency_norm&#34;"
209+
data_type="int_type"
210+
type="constant"/>
211+
<field name="growth_rate_tolerance"
212+
path="wavevector/eigenmode/growth_rate_tolerance"
213+
path_doc="wavevector(i1)/eigenmode(i2)/growth_rate_tolerance"
214+
documentation="Relative tolerance on the growth rate (convergence of the simulation)"
215+
data_type="FLT_0D"
216+
type="constant"
217+
units="-"/>
218+
</field>
219+
</field>
220+
</IDS>
221+
</IDSs>

imaspy/dd_helpers.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import re
77
import shutil
88
import subprocess
9-
from distutils.version import StrictVersion as V
9+
from packaging.version import Version as V
1010
from io import BytesIO
1111
from pathlib import Path
1212
from urllib.request import urlopen

0 commit comments

Comments
 (0)