Issue 713: Add commandline option for location of rfi files

rflx/cli.py

@@ -6,7 +6,7 @@
 from collections import defaultdict
 from multiprocessing import cpu_count
 from pathlib import Path
-from typing import Dict, List, Sequence, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Tuple, Union
 
 import librflxlang
 from pkg_resources import get_distribution
@@ -85,6 +85,9 @@ def main(argv: List[str]) -> Union[int, str]:
         help="ignore checksum aspects during code generation",
         action="store_true",
     )
+    parser_generate.add_argument(
+        "--integration-files-dir", help="directory for the .rfi files", type=Path
+    )
     parser_generate.add_argument(
         "files", metavar="SPECIFICATION_FILE", type=Path, nargs="*", help="specification file"
     )
@@ -249,7 +252,9 @@ def generate(args: argparse.Namespace) -> None:
     if not args.output_directory.is_dir():
         fail(f'directory not found: "{args.output_directory}"', Subsystem.CLI)
 
-    model, integration = parse(args.files, args.no_verification, args.workers)
+    model, integration = parse(
+        args.files, args.no_verification, args.workers, args.integration_files_dir
+    )
 
     generator = Generator(
         model,
@@ -267,9 +272,14 @@ def generate(args: argparse.Namespace) -> None:
 
 
 def parse(
-    files: Sequence[Path], skip_verification: bool = False, workers: int = 1
+    files: Sequence[Path],
+    skip_verification: bool = False,
+    workers: int = 1,
+    integration_files_dir: Optional[Path] = None,
 ) -> Tuple[Model, Integration]:
-    parser = Parser(skip_verification, cached=True, workers=workers)
+    parser = Parser(
+        skip_verification, cached=True, workers=workers, integration_files_dir=integration_files_dir
+    )
     error = RecordFluxError()
     present_files = []
 

rflx/integration.py

@@ -3,6 +3,8 @@
 
 from pydantic import BaseModel, Extra, Field, ValidationError
 from pydantic.types import ConstrainedInt
+from ruamel.yaml.error import MarkedYAMLError
+from ruamel.yaml.main import YAML
 
 from rflx.error import Location, RecordFluxError, Severity, Subsystem
 from rflx.identifier import ID
@@ -33,16 +35,32 @@ class Integration:
     def defaultsize(self) -> int:
         return 4096
 
-    def __init__(self) -> None:
+    def __init__(self, integration_files_dir: Optional[Path] = None) -> None:
         self._packages: Dict[str, IntegrationFile] = {}
-
-    def add_integration_file(self, filename: Path, file: object, error: RecordFluxError) -> None:
-        try:
-            self._packages[filename.stem] = IntegrationFile.parse_obj(file)
-        except ValidationError as e:
-            error.extend(
-                [(f"{e}", Subsystem.PARSER, Severity.ERROR, self._to_location(filename.stem))]
-            )
+        self._integration_files_dir = integration_files_dir
+
+    def load_integration_file(self, spec_file: Path, error: RecordFluxError) -> None:
+        integration_file = (
+            spec_file.with_suffix(".rfi")
+            if self._integration_files_dir is None
+            else self._integration_files_dir / (spec_file.stem + ".rfi")
+        )
+        if integration_file.exists():
+            yaml = YAML()
+            try:
+                content = yaml.load(integration_file)
+            except MarkedYAMLError as e:
+                location = Location(
+                    start=(
+                        (0, 0)
+                        if e.problem_mark is None
+                        else (e.problem_mark.line + 1, e.problem_mark.column + 1)
+                    ),
+                    source=integration_file,
+                )
+                error.extend([(str(e), Subsystem.PARSER, Severity.ERROR, location)])
+                return
+            self._add_integration_object(integration_file, content, error)
 
     def validate(self, model: Model, error: RecordFluxError) -> None:
         for package, integration_file in self._packages.items():
@@ -103,6 +121,14 @@ def get_size(self, session: ID, variable: ID, state: Optional[ID]) -> int:
             return buffer_size.local_[state_name][variable_name]
         return default_size
 
+    def _add_integration_object(self, filename: Path, file: object, error: RecordFluxError) -> None:
+        try:
+            self._packages[filename.stem] = IntegrationFile.parse_obj(file)
+        except ValidationError as e:
+            error.extend(
+                [(f"{e}", Subsystem.PARSER, Severity.ERROR, self._to_location(filename.stem))]
+            )
+
     @staticmethod
     def _to_location(package: str) -> Location:
         return Location(start=(0, 0), source=Path(package + ".rfi"))

rflx/specification/parser.py

@@ -8,8 +8,6 @@
 from typing import Callable, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type, Union
 
 import librflxlang as lang
-from ruamel.yaml.error import MarkedYAMLError
-from ruamel.yaml.main import YAML
 
 from rflx import expression as expr, model
 from rflx.error import Location, RecordFluxError, Severity, Subsystem, fail, warn
@@ -1424,7 +1422,11 @@ class SpecificationNode:
 
 class Parser:
     def __init__(
-        self, skip_verification: bool = False, cached: bool = False, workers: int = 1
+        self,
+        skip_verification: bool = False,
+        cached: bool = False,
+        workers: int = 1,
+        integration_files_dir: Optional[Path] = None,
     ) -> None:
         if skip_verification:
             warn("model verification skipped", Subsystem.MODEL)
@@ -1436,7 +1438,7 @@ def __init__(
             *model.INTERNAL_TYPES.values(),
         ]
         self.__sessions: List[model.Session] = []
-        self.__integration: Integration = Integration()
+        self.__integration: Integration = Integration(integration_files_dir)
         self.__cache = Cache(not skip_verification and cached)
 
     def __convert_unit(
@@ -1514,8 +1516,7 @@ def __parse_specfile(self, filename: Path, transitions: List[ID] = None) -> Reco
         unit = lang.AnalysisContext().get_from_file(str(filename))
         if diagnostics_to_error(unit.diagnostics, error, filename):
             return error
-        integration_file = filename.with_suffix(".rfi")
-        self._load_integration_file(integration_file, error)
+        self.__integration.load_integration_file(filename, error)
         if unit.root:
             assert isinstance(unit.root, lang.Specification)
             self.__convert_unit(error, unit.root, filename, transitions)
@@ -1582,24 +1583,6 @@ def create_model(self) -> model.Model:
     def get_integration(self) -> Integration:
         return self.__integration
 
-    def _load_integration_file(self, integration_file: Path, error: RecordFluxError) -> None:
-        if integration_file.exists():
-            yaml = YAML()
-            try:
-                content = yaml.load(integration_file)
-            except MarkedYAMLError as e:
-                location = Location(
-                    start=(
-                        (0, 0)
-                        if e.problem_mark is None
-                        else (e.problem_mark.line + 1, e.problem_mark.column + 1)
-                    ),
-                    source=integration_file,
-                )
-                error.extend([(str(e), Subsystem.PARSER, Severity.ERROR, location)])
-                return
-            self.__integration.add_integration_file(integration_file, content, error)
-
     @property
     def specifications(self) -> Dict[str, lang.Specification]:
         return {

tests/unit/integration_test.py

@@ -1,5 +1,6 @@
 import re
 from pathlib import Path
+from typing import Sequence
 
 import pytest
 from ruamel.yaml.main import YAML
@@ -110,7 +111,8 @@ def test_rfi_add_integration(rfi_content: str, match_error: str) -> None:
     error = RecordFluxError()
     integration = Integration()
     with pytest.raises(RecordFluxError, match=regex):
-        integration.add_integration_file(Path("test.rfi"), content, error)
+        # pylint: disable = protected-access
+        integration._add_integration_object(Path("test.rfi"), content, error)
         error.propagate()
 
 
@@ -124,11 +126,62 @@ def test_rfi_get_size() -> None:
         }
     }
     error = RecordFluxError()
-    integration.add_integration_file(Path("p.rfi"), session_object, error)
+    # pylint: disable = protected-access
+    integration._add_integration_object(Path("p.rfi"), session_object, error)
     error.propagate()
     assert integration.get_size(ID("P::S"), ID("x"), ID("S")) == 1024
     assert integration.get_size(ID("P::S"), ID("x"), ID("S")) == 1024
     assert integration.get_size(ID("P::S"), ID("x"), None) == 1024
     assert integration.get_size(ID("P::S2"), ID("x"), None) == 4096
     assert integration.get_size(ID("P::S"), ID("y"), None) == 2048
     assert integration.get_size(ID("P::S"), ID("y"), ID("S")) == 8192
+
+
+@pytest.mark.parametrize(
+    "content, error_msg, line, column",
+    [
+        ('"', ["while scanning a quoted scalar", "unexpected end of stream"], 1, 2),
+        ("Session: 1, Session : 1", ["mapping values are not allowed here"], 1, 21),
+        (
+            "Session: 1\nSession : 1",
+            ["while constructing a mapping", 'found duplicate key "Session" with value "1"'],
+            2,
+            1,
+        ),
+    ],
+)
+def test_load_integration_file(
+    tmp_path: Path, content: str, error_msg: Sequence[str], line: int, column: int
+) -> None:
+    test_rfi = tmp_path / "test.rfi"
+    test_rfi.write_text(content)
+    integration = Integration()
+    error = RecordFluxError()
+    regex = fr"^{test_rfi}:{line}:{column}: parser: error: "
+    for elt in error_msg:
+        regex += elt
+        regex += fr'.*in "{test_rfi}", line [0-9]+, column [0-9]+.*'
+    regex += "$"
+    compiled_regex = re.compile(regex, re.DOTALL)
+    with pytest.raises(RecordFluxError, match=compiled_regex):
+        integration.load_integration_file(test_rfi, error)
+        error.propagate()
+
+
+def test_load_integration_path(tmp_path: Path) -> None:
+    subfolder = tmp_path / "sub"
+    subfolder.mkdir()
+    test_rfi = subfolder / "test.rfi"
+    test_rfi.write_text("{ Session: { Session : { Buffer_Size : 0 }}}")
+    integration = Integration(integration_files_dir=subfolder)
+    error = RecordFluxError()
+    regex = re.compile(
+        (
+            r"test.rfi:0:0: parser: error: 1 validation error for IntegrationFile.*"
+            r"value is not a valid dict \(type=type_error.dict\)"
+        ),
+        re.DOTALL,
+    )
+    with pytest.raises(RecordFluxError, match=regex):
+        integration.load_integration_file(tmp_path / "test.rflx", error)
+        error.propagate()

tests/unit/specification/parser_test.py

@@ -1,6 +1,5 @@
 # pylint: disable=too-many-lines
 
-import re
 from itertools import zip_longest
 from pathlib import Path
 from typing import Any, Dict, Sequence
@@ -2803,35 +2802,3 @@ def test_parse_reserved_word_as_channel_name() -> None:
         end Test;
         """
     )
-
-
-@pytest.mark.parametrize(
-    "content, error_msg, line, column",
-    [
-        ('"', ["while scanning a quoted scalar", "unexpected end of stream"], 1, 2),
-        ("Session: 1, Session : 1", ["mapping values are not allowed here"], 1, 21),
-        (
-            "Session: 1\nSession : 1",
-            ["while constructing a mapping", 'found duplicate key "Session" with value "1"'],
-            2,
-            1,
-        ),
-    ],
-)
-def test_load_integration_file(
-    tmp_path: Path, content: str, error_msg: Sequence[str], line: int, column: int
-) -> None:
-    test_rfi = tmp_path / "test.rfi"
-    test_rfi.write_text(content)
-    p = parser.Parser()
-    error = RecordFluxError()
-    regex = fr"^{test_rfi}:{line}:{column}: parser: error: "
-    for elt in error_msg:
-        regex += elt
-        regex += fr'.*in "{test_rfi}", line [0-9]+, column [0-9]+.*'
-    regex += "$"
-    compiled_regex = re.compile(regex, re.DOTALL)
-    with pytest.raises(RecordFluxError, match=compiled_regex):
-        # pylint: disable = protected-access
-        p._load_integration_file(test_rfi, error)
-        error.propagate()