Use Jinja2 to parse layout files (editable packages)

conans/model/editable_cpp_info.py

@@ -1,11 +1,14 @@
 # coding=utf-8
 import os
 from collections import OrderedDict
+from io import StringIO
 
 from six.moves import configparser
 
 from conans.errors import ConanException
 from conans.model.ref import ConanFileReference
+from conans.util.files import load
+from conans.util.templates import render_layout_file
 
 DEFAULT_LAYOUT_FILE = "default"
 LAYOUTS_FOLDER = 'layouts'
@@ -14,19 +17,17 @@
 def get_editable_abs_path(path, cwd, cache_folder):
     # Check the layout file exists, is correct, and get its abs-path
     if path:
-        layout_abs_path = path if os.path.isabs(path) else os.path.normpath(os.path.join(cwd, path))
+        layout_abs_path = os.path.normpath(os.path.join(cwd, path))
         if not os.path.isfile(layout_abs_path):
             layout_abs_path = os.path.join(cache_folder, LAYOUTS_FOLDER, path)
         if not os.path.isfile(layout_abs_path):
             raise ConanException("Couldn't find layout file: %s" % path)
-        EditableLayout.load(layout_abs_path)  # Try if it loads ok
         return layout_abs_path
 
     # Default only in cache
-    layout_abs_path = os.path.join(cache_folder, LAYOUTS_FOLDER, DEFAULT_LAYOUT_FILE)
-    if os.path.isfile(layout_abs_path):
-        EditableLayout.load(layout_abs_path)
-        return layout_abs_path
+    layout_default_path = os.path.join(cache_folder, LAYOUTS_FOLDER, DEFAULT_LAYOUT_FILE)
+    if os.path.isfile(layout_default_path):
+        return layout_default_path
 
 
 class EditableLayout(object):
@@ -35,68 +36,76 @@ class EditableLayout(object):
     cpp_info_dirs = ['includedirs', 'libdirs', 'resdirs', 'bindirs', 'builddirs', 'srcdirs']
     folders = [BUILD_FOLDER, SOURCE_FOLDER]
 
-    def __init__(self, data, folders):
-        self._data = data
-        self._folders = folders
+    def __init__(self, filepath):
+        self._filepath = filepath
 
     def folder(self, ref, name, settings, options):
+        _, folders = self._load_data(ref, settings=settings, options=options)
         try:
-            path = self._folders.get(str(ref)) or self._folders.get(None) or {}
-            path = path[name]
+            path = folders.get(str(ref)) or folders.get(None) or {}
+            return path[name]
         except KeyError:
             return None
-        try:
-            return self._work_on_item(path, settings, options)
-        except Exception as e:
-            raise ConanException("Error getting fHolder '%s' from layout: %s" % (str(name), str(e)))
 
     @staticmethod
-    def load(filepath):
-        parser = configparser.ConfigParser(allow_no_value=True)
-        parser.optionxform = str
+    def _work_on_item(value):
+        value = value.replace('\\', '/')
+        return value
+
+    def _parse_layout_file(self, ref, settings, options):
+        content = load(self._filepath)
         try:
-            parser.read(filepath)
-        except configparser.Error as e:
-            raise ConanException("Error parsing layout file: %s\n%s" % (filepath, str(e)))
+            content = render_layout_file(content, ref=ref, settings=settings, options=options)
+
+            parser = configparser.ConfigParser(allow_no_value=True)
+            parser.optionxform = str
+            parser.readfp(StringIO(content))
+        except (configparser.Error, ConanException) as e:
+            raise ConanException("Error parsing layout file '%s' (for reference '%s')\n%s" %
+                                 (self._filepath, str(ref), str(e)))
+
+        return parser
+
+    def _load_data(self, ref, settings, options):
+        parser = self._parse_layout_file(ref, settings, options)
+
+        # Build a convenient data structure
         data = OrderedDict()
         folders = {}
         for section in parser.sections():
-            ref, section_name = section.rsplit(":", 1) if ':' in section else (None, section)
+            reference, section_name = section.rsplit(":", 1) if ':' in section else (None, section)
+
             if section_name in EditableLayout.folders:
                 items = [k for k, _ in parser.items(section)] or [""]
                 if len(items) > 1:
                     raise ConanException("'%s' with more than one value in layout file: %s"
-                                         % (section_name, filepath))
-                folders.setdefault(ref, {})[section_name] = items[0]
+                                         % (section_name, self._filepath))
+                folders.setdefault(reference, {})[section_name] = self._work_on_item(items[0])
                 continue
+
             if section_name not in EditableLayout.cpp_info_dirs:
                 raise ConanException("Wrong cpp_info field '%s' in layout file: %s"
-                                     % (section_name, filepath))
-            if ref:
+                                     % (section_name, self._filepath))
+            if reference:
                 try:
-                    r = ConanFileReference.loads(ref, validate=True)
+                    r = ConanFileReference.loads(reference, validate=True)
                     if r.revision:
                         raise ConanException("Don't provide revision in Editable layouts")
                 except ConanException:
                     raise ConanException("Wrong package reference '%s' in layout file: %s"
-                                         % (ref, filepath))
-            data.setdefault(ref, {})[section_name] = [k for k, _ in parser.items(section)]
-
-        return EditableLayout(data, folders)
-
-    @staticmethod
-    def _work_on_item(value, settings, options):
-        value = value.format(settings=settings, options=options)
-        value = value.replace('\\', '/')
-        return value
+                                         % (reference, self._filepath))
+            data.setdefault(reference, {})[section_name] =\
+                [self._work_on_item(k) for k, _ in parser.items(section)]
+        return data, folders
 
     def apply_to(self, ref, cpp_info, settings=None, options=None):
-        d = self._data
-        data = d.get(str(ref)) or d.get(None) or {}
+        data, _ = self._load_data(ref, settings=settings, options=options)
+
+        # Apply the data to the cpp_info
+        data = data.get(str(ref)) or data.get(None) or {}
 
         try:
             for key, items in data.items():
-                setattr(cpp_info, key, [self._work_on_item(item, settings, options)
-                                        for item in items])
+                setattr(cpp_info, key, items)
         except Exception as e:
             raise ConanException("Error applying layout in '%s': %s" % (str(ref), str(e)))

conans/model/workspace.py

@@ -19,8 +19,7 @@ def __init__(self, base_folder, data, cache, ws_layout, ws_generators, ref):
         self._conanfile_folder = data.pop("path", None)  # The folder with the conanfile
         layout = data.pop("layout", None)
         if layout:
-            self.layout = get_editable_abs_path(layout, self._base_folder,
-                                                cache.conan_folder)
+            self.layout = get_editable_abs_path(layout, self._base_folder, cache.conan_folder)
         else:
             self.layout = ws_layout
 
@@ -52,6 +51,7 @@ def generate(self, cwd, graph, output):
                 ws_pkg = self._workspace_packages[ref]
                 layout = self._cache.package_layout(ref)
                 editable = layout.editable_cpp_info()
+
                 conanfile = node.conanfile
                 build = editable.folder(ref, EditableLayout.BUILD_FOLDER, conanfile.settings,
                                         conanfile.options)
@@ -73,6 +73,7 @@ def generate(self, cwd, graph, output):
                                     % (ref.name, ref.name))
                 else:
                     output.warn("CMake workspace: cannot 'add_subdirectory()'")
+
             if add_subdirs:
                 cmake += "macro(conan_workspace_subdirectories)\n"
                 cmake += add_subdirs

conans/paths/package_layouts/package_editable_layout.py

@@ -30,7 +30,7 @@ def conanfile(self):
     def editable_cpp_info(self):
         if self._layout_file:
             if os.path.isfile(self._layout_file):
-                return EditableLayout.load(self._layout_file)
+                return EditableLayout(self._layout_file)
             else:
                 raise ConanException("Layout file not found: %s" % self._layout_file)
 

conans/requirements.txt

@@ -13,3 +13,4 @@ pygments>=2.0, <3.0
 astroid>=1.6.5
 deprecation>=2.0, <2.1
 tqdm>=4.28.1, <5
+Jinja2==2.10

conans/test/functional/editable/consume_header_only_test.py

@@ -2,16 +2,15 @@
 
 import os
 import tempfile
-import unittest
 import textwrap
+import unittest
 
 from parameterized import parameterized
 
-
+from conans.model.editable_cpp_info import DEFAULT_LAYOUT_FILE, LAYOUTS_FOLDER
 from conans.test import CONAN_TEST_FOLDER
 from conans.test.utils.tools import TestClient
 from conans.util.files import save
-from conans.model.editable_cpp_info import DEFAULT_LAYOUT_FILE, LAYOUTS_FOLDER
 
 
 class HeaderOnlyLibTestClient(TestClient):

conans/test/functional/editable/consume_settings_and_options_test.py

@@ -4,12 +4,13 @@
 import os
 import tempfile
 import unittest
+
 from parameterized import parameterized
 
+from conans.model.editable_cpp_info import DEFAULT_LAYOUT_FILE, LAYOUTS_FOLDER
 from conans.test import CONAN_TEST_FOLDER
 from conans.test.utils.tools import TestClient
 from conans.util.files import save
-from conans.model.editable_cpp_info import DEFAULT_LAYOUT_FILE, LAYOUTS_FOLDER
 
 
 class HeaderOnlyLibTestClient(TestClient):
@@ -48,7 +49,7 @@ def package_info(self):
     """
 
     conan_package_layout = """
-[{namespace}includedirs]
+[%sincludedirs]
 src/include/{{settings.build_type}}/{{options.shared}}
 """
 
@@ -67,11 +68,11 @@ def __init__(self, use_repo_file, *args, **kwargs):
                  }
 
         if use_repo_file:
-            files["mylayout"] = self.conan_package_layout.format(namespace="")
+            files["mylayout"] = self.conan_package_layout % ""
         else:
             file_path = os.path.join(self.cache.conan_folder, LAYOUTS_FOLDER, DEFAULT_LAYOUT_FILE)
             save(file_path,
-                 self.conan_package_layout.format(namespace="MyLib/0.1@user/editable:"))
+                 self.conan_package_layout % "MyLib/0.1@user/editable:")
 
         self.save(files)
 

conans/test/functional/editable/layouts_test.py

@@ -5,10 +5,10 @@
 import textwrap
 import unittest
 
-from conans.test.utils.tools import TestClient
-from conans.util.files import load, save_files, save
 from conans.model.editable_cpp_info import LAYOUTS_FOLDER
 from conans.test.utils.test_files import temp_folder
+from conans.test.utils.tools import TestClient
+from conans.util.files import load, save_files, save
 
 
 class LayoutTest(unittest.TestCase):
@@ -187,7 +187,7 @@ class Pkg(ConanFile):
             """)
         layout_repo = textwrap.dedent("""
             [includedirs]
-            include_{settings.build_type}
+            include_{{settings.build_type}}
             """)
 
         client.save({"conanfile.py": conanfile,
@@ -200,8 +200,9 @@ class Pkg(ConanFile):
             """)
         client2.save({"conanfile.txt": consumer})
         client2.run("install . -g cmake -s build_type=Debug", assert_error=True)
-        self.assertIn("ERROR: Error applying layout in 'mytool/0.1@user/testing': "
-                      "'settings.build_type' doesn't exist", client2.out)
+        self.assertIn("ERROR: Error parsing layout file '{}' (for reference "
+                      "'mytool/0.1@user/testing')\n'settings.build_type' doesn't exist".format(
+            os.path.join(client.current_folder, 'layout')), client2.out)
 
         # Now add settings to conanfile
         client.save({"conanfile.py": conanfile.replace("pass", 'settings = "build_type"')})

conans/test/functional/editable/link_create_test.py

@@ -48,6 +48,19 @@ def test_editable_list_search(self):
         t.run("search")
         self.assertIn("lib/version@user/name", t.out)
 
+    def test_editable_list_search(self):
+        ref = ConanFileReference.loads('lib/version@user/name')
+        t = TestClient()
+        t.save(files={'conanfile.py': self.conanfile})
+        t.run('editable add . {}'.format(ref))
+        t.run("editable list")
+        self.assertIn("lib/version@user/name", t.out)
+        self.assertIn("    Layout: None", t.out)
+        self.assertIn("    Path:", t.out)
+
+        t.run("search")
+        self.assertIn("lib/version@user/name", t.out)
+
     def test_install_wrong_reference(self):
         ref = ConanFileReference.loads('lib/version@user/name')
 

conans/test/integration/workspace_test.py

@@ -222,13 +222,13 @@ def files(name, depend=None):
             """)
         layout = dedent("""
             [build_folder]
-            build/{settings.build_type}
+            build/{{settings.build_type}}
 
             [includedirs]
             src
 
             [libdirs]
-            build/{settings.build_type}/lib
+            build/{{settings.build_type}}/lib
             """)
         client.save({"conanws.yml": project,
                      "layout": layout})
@@ -295,7 +295,7 @@ def files(name, depend=None):
             """)
         layout = dedent("""
             [build_folder]
-            build/{settings.build_type}
+            build/{{settings.build_type}}
 
             [source_folder]
             src
@@ -304,7 +304,7 @@ def files(name, depend=None):
             src
 
             [libdirs]
-            build/{settings.build_type}/lib
+            build/{{settings.build_type}}/lib
             """)
 
         metacmake = dedent("""
@@ -422,7 +422,7 @@ def files(name, depend=None):
             src
 
             [libdirs]
-            build/{settings.build_type}
+            build/{{settings.build_type}}
             """)
         metacmake = dedent("""
             cmake_minimum_required(VERSION 3.3)

conans/test/unittests/model/editable_cpp_info/apply_test.py

@@ -6,12 +6,11 @@
 import textwrap
 import unittest
 
-from conans.model.editable_cpp_info import EditableLayout
 from conans.model.build_info import CppInfo
+from conans.model.editable_cpp_info import EditableLayout
+from conans.model.ref import ConanFileReference
 from conans.test.utils.test_files import temp_folder
 from conans.util.files import save
-from conans.model.ref import ConanFileReference
-
 
 base_content = textwrap.dedent("""\
     [{namespace}includedirs]
@@ -35,16 +34,16 @@ def setUp(self):
         self.test_folder = temp_folder()
         self.layout_filepath = os.path.join(self.test_folder, "layout")
         self.ref = ConanFileReference.loads("libA/0.1@user/channel")
+        self.editable_cpp_info = EditableLayout(self.layout_filepath)
 
     def tearDown(self):
         shutil.rmtree(self.test_folder)
 
     def test_require_no_namespace(self):
         content = base_content.format(namespace="", path_prefix="")
         save(self.layout_filepath, content)
-        editable_cpp_info = EditableLayout.load(self.layout_filepath)
         cpp_info = CppInfo(None)
-        editable_cpp_info.apply_to(self.ref, cpp_info, settings=None, options=None)
+        self.editable_cpp_info.apply_to(self.ref, cpp_info, settings=None, options=None)
         self.assertListEqual(cpp_info.includedirs, ['dirs/includedirs'])
         self.assertListEqual(cpp_info.libdirs, ['dirs/libdirs'])
         self.assertListEqual(cpp_info.resdirs, ['dirs/resdirs'])
@@ -57,9 +56,8 @@ def test_require_namespace(self):
             base_content.format(namespace="libA/0.1@user/channel:", path_prefix="libA/")
             ])
         save(self.layout_filepath, content)
-        editable_cpp_info = EditableLayout.load(self.layout_filepath)
         cpp_info = CppInfo(None)
-        editable_cpp_info.apply_to(self.ref, cpp_info, settings=None, options=None)
+        self.editable_cpp_info.apply_to(self.ref, cpp_info, settings=None, options=None)
         self.assertListEqual(cpp_info.includedirs, ['libA/dirs/includedirs'])
         self.assertListEqual(cpp_info.libdirs, ['libA/dirs/libdirs'])
         self.assertListEqual(cpp_info.resdirs, ['libA/dirs/resdirs'])
@@ -68,7 +66,7 @@ def test_require_namespace(self):
 
         cpp_info = CppInfo(None)
         other = ConanFileReference.loads("other/0.1@user/channel")
-        editable_cpp_info.apply_to(other, cpp_info, settings=None, options=None)
+        self.editable_cpp_info.apply_to(other, cpp_info, settings=None, options=None)
         self.assertListEqual(cpp_info.includedirs, ['dirs/includedirs'])
         self.assertListEqual(cpp_info.libdirs, ['dirs/libdirs'])
         self.assertListEqual(cpp_info.resdirs, ['dirs/resdirs'])