From 47089745eb0105670b39445d0f20fdf07982407d Mon Sep 17 00:00:00 2001 From: Arrielle Opotowsky Date: Wed, 28 Jun 2023 17:12:17 -0500 Subject: [PATCH 1/4] update _processIncludes to handle src that is StringIO --- armi/utils/textProcessors.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/armi/utils/textProcessors.py b/armi/utils/textProcessors.py index 5108efb70..e15003fd0 100644 --- a/armi/utils/textProcessors.py +++ b/armi/utils/textProcessors.py @@ -58,10 +58,10 @@ def __str__(self): def _processIncludes( - src, + src: Union[TextIO, pathlib.Path], out, includes: List[Tuple[pathlib.Path, FileMark]], - root, + root: pathlib.Path, indentation=0, currentFile="", ): @@ -88,7 +88,13 @@ def _beginningOfContent(line: str) -> int: return 0 indentSpace = " " * indentation - for i, line in enumerate(src.readlines()): + if hasattr(src, "getvalue"): + # assume stringIO + lines = [l + "\n" for l in src.getvalue().split("\n")] + else: + # assume file stream or TextIOBase, and it has a readlines attr + lines = src.readlines() + for i, line in enumerate(lines): leadingSpace = indentSpace if i > 0 else "" m = _INCLUDE_RE.match(line) if m: From 1328833d6f484cdb035f13739d4bd12ca498a566 Mon Sep 17 00:00:00 2001 From: Arrielle Opotowsky Date: Fri, 14 Jul 2023 14:21:11 -0500 Subject: [PATCH 2/4] Add stringIO type to param def --- armi/utils/textProcessors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/armi/utils/textProcessors.py b/armi/utils/textProcessors.py index e15003fd0..b498f45b0 100644 --- a/armi/utils/textProcessors.py +++ b/armi/utils/textProcessors.py @@ -140,7 +140,7 @@ def resolveMarkupInclusions( Parameters ---------- - src : TextIOBase or Path + src : StringIO or TextIOBase/Path If a Path is provided, read text from there. If is stream is provided, consume text from the stream. If a stream is provided, ``root`` must also be provided. root : Optional Path From ade18304b54f0aa8bf0631002d905a237611c96a Mon Sep 17 00:00:00 2001 From: Arrielle Opotowsky Date: Fri, 14 Jul 2023 14:21:47 -0500 Subject: [PATCH 3/4] Write a test that handles stringIO as input. test files need small indentation updates --- .../utils/tests/resources/lower/includeA.yaml | 4 ++-- armi/utils/tests/resources/root.yaml | 12 +++++------ armi/utils/tests/test_textProcessors.py | 21 +++++++++++++++++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/armi/utils/tests/resources/lower/includeA.yaml b/armi/utils/tests/resources/lower/includeA.yaml index 74ae1e984..39e14f1d8 100644 --- a/armi/utils/tests/resources/lower/includeA.yaml +++ b/armi/utils/tests/resources/lower/includeA.yaml @@ -1,4 +1,4 @@ full_name: Jennifer Person # some comment in includeA -children: - !include includeB.yaml +children: + !include includeB.yaml diff --git a/armi/utils/tests/resources/root.yaml b/armi/utils/tests/resources/root.yaml index e5b07d370..e37c2dfb9 100644 --- a/armi/utils/tests/resources/root.yaml +++ b/armi/utils/tests/resources/root.yaml @@ -1,10 +1,10 @@ # Behold, the Person family bobby: &bobby - full_name: Robert Person + full_name: Robert Person billy: - full_name: William Person - # comment - children: - - *bobby - - !include lower/includeA.yaml + full_name: William Person + # comment + children: + - *bobby + - !include lower/includeA.yaml diff --git a/armi/utils/tests/test_textProcessors.py b/armi/utils/tests/test_textProcessors.py index 9df369c93..846676521 100644 --- a/armi/utils/tests/test_textProcessors.py +++ b/armi/utils/tests/test_textProcessors.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """Tests for functions in textProcessors.py.""" +from io import StringIO, TextIOWrapper, TextIOBase import os import pathlib -import unittest - import ruamel +import unittest from armi.utils import textProcessors @@ -60,6 +60,23 @@ def test_resolveIncludes(self): self.assertTrue(commentFound) self.assertTrue(anchorFound) + def test_resolveIncludes_StringIO(self): + """Tests that resolveMarkupInclusions handles StringIO input""" + yaml = ruamel.yaml.YAML() + with open(os.path.join(RES_DIR, "root.yaml")) as f: + loadedYaml = yaml.load(f) + stringIO = StringIO() + yaml.dump(loadedYaml, stringIO) + resolved = textProcessors.resolveMarkupInclusions( + src=stringIO, root=pathlib.Path(RES_DIR) + ) + with open(os.path.join(RES_DIR, "root.yaml")) as f: + expected = textProcessors.resolveMarkupInclusions( + f, root=pathlib.Path(RES_DIR) + ) + # strip it because one method gives an extra newline we don't care about + self.assertEqual(resolved.getvalue().strip(), expected.getvalue().strip()) + def test_findIncludes(self): includes = textProcessors.findYamlInclusions( pathlib.Path(RES_DIR) / "root.yaml" From 7641b95a1b3807a5c7dda014ab24e8a0ff81f787 Mon Sep 17 00:00:00 2001 From: Arrielle Opotowsky Date: Fri, 14 Jul 2023 14:23:24 -0500 Subject: [PATCH 4/4] release notes --- doc/release/0.2.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release/0.2.rst b/doc/release/0.2.rst index 22e1ff300..8107f41c6 100644 --- a/doc/release/0.2.rst +++ b/doc/release/0.2.rst @@ -15,6 +15,7 @@ What's new in ARMI Bug fixes --------- +#. Fix _processIncludes to handle StringIO input (`PR#1333 `_) #. TBD ARMI v0.2.8