Merge pull request #435 from fast-aircraft-design/datafile-enhancement

Datafile enhancement
fast-aircraft-design · May 18, 2022 · 9af6c89 · 9af6c89
2 parents fdc21bb + 93424f5
commit 9af6c89
Show file tree

Hide file tree

Showing 8 changed files with 290 additions and 89 deletions.
diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml
@@ -2,6 +2,7 @@ name: Build PyPI package and Binder image
 
 on:
   workflow_dispatch:
+  workflow_call:
   release:
     types: [ published ]
 

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -1,13 +1,14 @@
 name: Tests
 
 on:
+  workflow_dispatch:
+  workflow_call:
   push:
     branches:
       - '**'
     tags-ignore:
       - '**'
 
-
 jobs:
   tests:
     runs-on: ${{ matrix.os }}
@@ -18,33 +19,21 @@ jobs:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
 
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v3
 
-      - name: Install Poetry
-        run: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
-        shell: bash
-
-      - name: Poetry path
-        run: echo "$HOME/.poetry/bin" >> $GITHUB_PATH
-        shell: bash
+      - name: Install poetry
+        run: pipx install poetry
 
-      - name: Sets unique cache directory for Poetry
-        run: poetry config cache-dir ~/poetry_cache
-        shell: bash
-
-      - uses: actions/cache@v2
-        if: ${{ ! startsWith(runner.os, 'Windows') }} # With time, caching on Windows eventually ends crashing.
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
         with:
-          path: ~/poetry_cache
-          key: ${{ runner.os }}-Py${{ matrix.python-version }}-pypoetry-${{ hashFiles('**/poetry.lock') }}
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
 
       - name: Activate environment and install dependencies
-        run: poetry install
-        shell: bash
+        run: |
+          poetry env use ${{ matrix.python-version }}
+          poetry install
 
       - name: Check with Black
         run: |
@@ -84,6 +73,6 @@ jobs:
         shell: bash
 
       - name: Notebook tests
-#        if: ${{ github.event_name == 'pull_request' || contains(github.event.head_commit.message, '[test nb]') || github.ref == 'refs/heads/master' }}
+        #        if: ${{ github.event_name == 'pull_request' || contains(github.event.head_commit.message, '[test nb]') || github.ref == 'refs/heads/master' }}
         run: poetry run pytest --no-cov --nbval-lax -p no:python src/fastoad/notebooks
         shell: bash
diff --git a/.github/workflows/watchman_tests.yml b/.github/workflows/watchman_tests.yml
@@ -4,13 +4,15 @@ name: Watchman Tests
 # It will warn developers if any update of a dependency broke something.
 
 on:
+  workflow_dispatch:
+  workflow_call:
   push:
     branches: [ master ]
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ master ]
   schedule:
-    - cron: '12 5 * * *'
+    - cron: '12 4 * * *'
 
 
 jobs:
@@ -22,9 +24,9 @@ jobs:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/docs/documentation/data.xml b/docs/documentation/data.xml
@@ -0,0 +1,17 @@
+<FASTOAD_model>
+  <data>
+    <geometry>
+      <wing>
+        <area units="m**2">150.0</area>
+      </wing>
+    </geometry>
+    <weight>
+      <fuselage>
+        <mass units="kg">10000.0</mass>
+        <CG>
+          <x units="m">20.0</x>
+        </CG>
+      </fuselage>
+    </weight>
+  </data>
+</FASTOAD_model>
diff --git a/docs/documentation/variables.rst b/docs/documentation/variables.rst
@@ -17,6 +17,9 @@ available for the whole process.
 The list of variable names and descriptions for a given problem can be obtained from command line (see
 :ref:`get-variable-list`).
 
+.. contents::
+   :local:
+
 
 ***************
 Variable naming
@@ -42,28 +45,134 @@ The other path elements depend of the variable. The number of path elements is n
 Serialization
 ***************
 
+File format
+***********
+
 For writing input and output files, FAST-OAD relies on the path in the variable names.
 
-For example, for the three variables above, the matching part in XML file will be:
+For instance, the variable name :code:`data:geometry:wing:area` will be split according
+to colons :code:`:` and each part of the name will become a level in the XML hierarchy:
+
+.. code-block:: xml
+
+        <data>
+            <geometry>
+                <wing>
+                    <area units="m**2">
+                        150.0
+                    </area>
+                </wing>
+            </geometry>
+        </data>
+
+
+A complete file that would contain the three above-mentioned variables will be as following:
+
+
+.. code-block:: xml
+
+    <FASTOAD_model>
+        <data>
+            <geometry>
+                <wing>
+                    <area units="m**2">150.0</area>
+                </wing>
+            </geometry>
+            <weight>
+                <fuselage>
+                    <mass units="kg">10000.0</mass>
+                    <CG>
+                        <x units="m">20.0</x>
+                    </CG>
+                </fuselage>
+            </weight>
+        </data>
+    </FASTOAD_model>
+
+.. note::
+
+    Units are given as a string according to
+    `OpenMDAO units definitions <http://openmdao.org/twodocs/versions/latest/features/units.html>`_
+
+.. note::
+
+    XML requires a unique root element for containing all other ones. Its name can be
+    freely chose, but it is `FASTOAD_model` in files written by FAST-OAD
+
+
+FAST-OAD API
+************
+
+FAST-OAD proposes a convenient way to read/write such files in Python, through the
+:class:`~fastoad.io.variable_io.DataFile` class.
+
+Provided that above file is named :code:`data.xml`, following commands apply:
+
+.. doctest::
+
+    >>> import fastoad.api as oad
+    >>> # ---------------------------------
+    >>> datafile = oad.DataFile("./data.xml")
+    >>> # Getting information
+    >>> datafile.names()
+    ['data:geometry:wing:area', 'data:weight:fuselage:mass', 'data:weight:fuselage:CG:x']
+    >>> len(datafile)
+    3
+    >>> datafile["data:geometry:wing:area"].value
+    [150.0]
+    >>> datafile["data:geometry:wing:area"].units
+    'm**2'
+    >>> # ---------------------------------
+    >>> # Writing data
+    >>> datafile.save()
+    >>> # ---------------------------------
+    >>> # Modifying data
+    >>> datafile["data:geometry:wing:area"].value = 120.0  # no need to provide list or numpy array for scalar values.
+    >>> datafile["data:geometry:wing:area"].value
+    120.0
+    >>> # ---------------------------------
+    >>> # Adding data
+    >>> fuselage_length = oad.Variable("data:geometry:fuselage:length", val=35.0, units="m")
+    >>> datafile.append(fuselage_length)
+    >>> # or ...
+    >>> datafile["data:geometry:wing:mass"] = dict(val=10500.0, units="kg") # will replace previous definition
+    >>> datafile.names()
+    ['data:geometry:wing:area', 'data:weight:fuselage:mass', 'data:weight:fuselage:CG:x', 'data:geometry:fuselage:length', 'data:geometry:wing:mass']
+    >>> # ---------------------------------
+    >>> # Removing data
+    >>> del datafile["data:weight:fuselage:CG:x"]
+    >>> datafile.names()
+    ['data:geometry:wing:area', 'data:weight:fuselage:mass', 'data:geometry:fuselage:length', 'data:geometry:wing:mass']
+    >>> # ---------------------------------
+    >>> # Writing to another file
+    >>> datafile.save_as("./new_data.xml", overwrite=True)
+    >>> datafile.file_path  # The object is now associated to the new path
+    './new_data.xml'
+
+After running these lines of code, the generated file :code:`new_data.xml` contains:
 
 .. code-block:: xml
 
-    <data>
-        <geometry>
-            <wing>
-                <area units="m**2">150.0</area>
-            </wing>
-        </geometry>
-        <weight>
-            <fuselage>
-                <mass units="kg">10000.0</mass>
-                <CG>
-                    <x units="m">20.0</x>
-                </CG>
-            </fuselage>
-        </weight>
-    </data>
-
-**Note**: *Units are given as a string according to* `OpenMDAO units definitions <http://openmdao.org/twodocs/versions/latest/features/units.html>`_
+    <FASTOAD_model>
+        <data>
+            <geometry>
+                <fuselage>
+                    <length units="m">35.0</length>
+                </fuselage>
+                <wing>
+                    <area units="m**2">120.0</area>
+                    <mass units="kg">10500.0</mass>
+                </wing>
+            </geometry>
+            <weight>
+                <fuselage>
+                    <mass units="kg">10000.0</mass>
+                </fuselage>
+            </weight>
+        </data>
+    </FASTOAD_model>
+
+
+
 
 .. [#] see :ref:`add-modules-register-systems`
diff --git a/src/fastoad/io/tests/test_data_file.py b/src/fastoad/io/tests/test_data_file.py
@@ -1,5 +1,5 @@
 #  This file is part of FAST-OAD : A framework for rapid Overall Aircraft Design
-#  Copyright (C) 2021 ONERA & ISAE-SUPAERO
+#  Copyright (C) 2022 ONERA & ISAE-SUPAERO
 #  FAST is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
 #  the Free Software Foundation, either version 3 of the License, or
@@ -46,18 +46,39 @@ def write_variables(self, data_source: Union[str, IO], variables: VariableList):
         self.variables.update(variables, add_variables=True)
 
 
-def test(cleanup):
+def test_DataFile(cleanup):
+    variables_ref = VariableList([Variable("data:foo", value=5), Variable("data:bar", value=10)])
+
     file_path = pth.join(RESULTS_FOLDER_PATH, "dummy_data_file.xml")
-    variables_1 = DataFile(file_path)
-    assert len(variables_1) == 0
+    with pytest.raises(FileNotFoundError) as exc_info:
+        _ = DataFile(file_path)
+    assert exc_info.value.args[0] == f'File "{file_path}" is unavailable for reading.'
+
+    data_file_1 = DataFile()
+    assert len(data_file_1) == 0
 
-    variables_1.update(
-        VariableList([Variable("data:foo", value=5), Variable("data:bar", value=10)]),
+    data_file_1.update(
+        variables_ref,
         add_variables=True,
     )
-    variables_1.save()
+    assert data_file_1.file_path is None
+    with pytest.raises(FileNotFoundError):
+        _ = data_file_1.save()
+    data_file_1.save_as(file_path)
+    assert data_file_1.file_path == file_path
+
+    data_file_2 = DataFile(file_path)
+    assert len(data_file_2) == 2
+
+    assert set(data_file_2) == set(variables_ref)
 
-    variables_2 = DataFile(file_path)
-    assert len(variables_2) == 2
+    # Test from_* methods
+    ivc = variables_ref.to_ivc()
+    data_file_3 = DataFile.from_ivc(ivc)
+    assert isinstance(data_file_3, DataFile)
+    assert set(data_file_3) == set(variables_ref)
 
-    assert set(variables_2) == set(variables_1)
+    df = variables_ref.to_dataframe()
+    data_file_4 = DataFile.from_dataframe(df)
+    assert isinstance(data_file_4, DataFile)
+    assert set(data_file_4) == set(variables_ref)