Refactor and introduce get_tool_options in flow

Before this change, there was no meaningful way to use flow options
because it was not possible to query or set tool options for tools
outside of the ones in the static FLOW list.

This patch separates the process of applying options into two steps.
A user can first use get_flow_options to get flow options and then
use get_tool_options with the preferred flow options set, to get
all tool options for a configured flow.

The redefinition of get_flow_options to only return flow options
(and not also tool options like previously) is an API break but will
hopefully not cause too much trouble.

edalize/flows/edaflow.py

@@ -73,6 +73,9 @@ def merge_dict(d1, d2):
 
 
 class Edaflow(object):
+
+    FLOW = []
+
     @classmethod
     def get_flow_options(cls):
         flow_opts = cls.FLOW_OPTIONS.copy()
@@ -90,6 +93,36 @@ def get_flow_options(cls):
                     flow_opts[opt_name]["tool"] = tool_name
         return flow_opts
 
+    @classmethod
+    def get_tool_options(cls, flow_options):
+        return {}
+
+    # Takes a list of tool names and a dict of pre-defined tool options
+    # Imports the tool class for every tool in the list, extracts their
+    # tool options and return then all, except for the ones listed in
+    # flow_defined_tool_options
+    @classmethod
+    def get_filtered_tool_options(cls, tools, flow_defined_tool_options):
+        tool_opts = {}
+        for tool_name in tools:
+
+            # Get available tool options from each tool in the list
+            try:
+                class_tool_options = getattr(
+                    import_module(f"edalize.tools.{tool_name}"), tool_name.capitalize()
+                ).get_tool_options()
+            except ModuleNotFoundError:
+                raise RuntimeError(f"Could not find tool '{tool_name}'")
+            # Add them to the dict unless they are already set by the flow
+            filtered_tool_options = flow_defined_tool_options.get(tool_name, {})
+            for opt_name in class_tool_options:
+                # Filter out tool options that are already set by the flow
+                if not opt_name in filtered_tool_options:
+                    tool_opts[opt_name] = class_tool_options[opt_name]
+                    tool_opts[opt_name]["tool"] = tool_name
+
+        return tool_opts
+
     def extract_flow_options(self):
         # Extract flow options from the EDAM
         flow_options = {}
@@ -106,7 +139,7 @@ def extract_flow_options(self):
     def extract_tool_options(self):
         tool_options = {}
         edam_flow_opts = self.edam.get("flow_options", {})
-        for (tool_name, next_nodes, flow_defined_tool_options) in self.FLOW:
+        for (tool_name, next_nodes, flow_defined_tool_options) in self.flow:
 
             # Get the tool class
             ToolClass = getattr(
@@ -130,7 +163,7 @@ def extract_tool_options(self):
     def build_tool_graph(self):
         # Instantiate the tools
         nodes = {}
-        for (tool_name, next_nodes, flow_defined_tool_options) in self.FLOW:
+        for (tool_name, next_nodes, flow_defined_tool_options) in self.flow:
 
             # Instantiate the tool class
             tool_inst = getattr(
@@ -201,13 +234,19 @@ def add_scripts(self, depends, hook_name):
             last_script = script["name"]
         self.commands.add([], [hook_name], [last_script])
 
+    def configure_flow(self, flow_options):
+        return self.FLOW
+
     def __init__(self, edam, work_root, verbose=False):
         self.edam = edam
+        self.commands = EdaCommands()
 
         # Extract all options that affects the flow rather than
         # just a single tool
         self.flow_options = self.extract_flow_options()
 
+        self.flow = self.configure_flow(self.flow_options)
+
         # Rearrange tool_options so that each tool gets their
         # own tool_options
         self.extract_tool_options()
@@ -216,7 +255,6 @@ def __init__(self, edam, work_root, verbose=False):
 
         self.stdout = None
         self.stderr = None
-        self.commands = EdaCommands()
 
     def set_run_command(self):
         self.commands.add([], ["run"], ["pre_run"])

edalize/flows/icestorm.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
 import os.path
+from importlib import import_module
 
 from edalize.flows.edaflow import Edaflow
 
@@ -12,13 +13,10 @@ class Icestorm(Edaflow):
 
     argtypes = ["vlogdefine", "vlogparam"]
 
-    FLOW = [
-        ("yosys", ["nextpnr"], {"arch": "ice40", "output_format": "json"}),
-        ("nextpnr", ["icepack", "icetime"], {"arch": "ice40"}),
-        ("icepack", [], {}),
-        ("icetime", [], {}),
-        #        ('icebox_stat', [], {}),
-    ]
+    FLOW_DEFINED_TOOL_OPTIONS = {
+        "yosys": {"arch": "ice40", "output_format": "json"},
+        "nextpnr": {"arch": "ice40"},
+    }
 
     FLOW_OPTIONS = {
         "frontends": {
@@ -32,26 +30,43 @@ class Icestorm(Edaflow):
         },
     }
 
-    def build_tool_graph(self):
-        # FIXME: This makes an assumption that the first tool in self.FLOW is
-        # a single entry point to the flow
-        next_tool = self.FLOW[0][0]
+    @classmethod
+    def get_flow_options(cls):
+        return cls.FLOW_OPTIONS.copy()
 
-        for frontend in reversed(self.flow_options.get("frontends", [])):
-            self.FLOW[0:0] = [(frontend, [next_tool], {})]
-            next_tool = frontend
-        return super().build_tool_graph()
+    @classmethod
+    def get_tool_options(cls, flow_options):
+        flow = flow_options.get("frontends", [])
+        flow += ["yosys"]
+        pnr = flow_options.get("pnr", "next")
+        if pnr == "next":
+            flow += ["nextpnr", "icepack", "icetime"]
 
-    def configure_tools(self, nodes):
-        super().configure_tools(nodes)
+        return cls.get_filtered_tool_options(flow, cls.FLOW_DEFINED_TOOL_OPTIONS)
+
+    def extract_flow_options(self):
+        return {
+            k: v
+            for (k, v) in self.edam.get("flow_options", {}).items()
+            if k in self.get_flow_options()
+        }
 
-        # Set output from syntheis or pnr as default target depending on
-        # value of pnr flow option
+    def configure_flow(self, flow_options):
         name = self.edam["name"]
-        pnr = self.flow_options.get("pnr", "next")
+        # Check whether to run nextpnr or stop after synthesis
+        # and set output from syntheis or pnr as default target depending
+        # on value of pnr flow option
+        pnr = flow_options.get("pnr", "next")
         if pnr == "next":
+            flow = [
+                ("yosys", ["nextpnr"], {"arch": "ice40", "output_format": "json"}),
+                ("nextpnr", ["icepack", "icetime"], {"arch": "ice40"}),
+                ("icepack", [], {}),
+                ("icetime", [], {}),
+            ]
             self.commands.set_default_target(name + ".bin")
         elif pnr == "none":
+            flow = [("yosys", [], {"arch": "ice40", "output_format": "json"})]
             self.commands.set_default_target(name + ".json")
         else:
             raise RuntimeError(
@@ -60,6 +75,19 @@ def configure_tools(self, nodes):
                 )
             )
 
+        # Add any user-specified frontends to the flow
+        next_tool = "yosys"
+
+        for frontend in reversed(flow_options.get("frontends", [])):
+            flow[0:0] = [(frontend, [next_tool], {})]
+            next_tool = frontend
+
+        return flow
+
+    def configure_tools(self, nodes):
+        super().configure_tools(nodes)
+
+        name = self.edam["name"]
         # Slot in statistics command which doesn't have an EdaTool class
         depends = name + ".asc"
         targets = name + ".stat"

edalize/flows/lint.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
 import os.path
+from importlib import import_module
 
 from edalize.flows.edaflow import Edaflow
 
@@ -12,10 +13,10 @@ class Lint(Edaflow):
 
     argtypes = ["vlogdefine", "vlogparam"]
 
-    FLOW = [
-        ("verilator", [], {"mode": "lint-only", "exe": "false", "make_options": []}),
+    FLOW_DEFINED_TOOL_OPTIONS = {
+        "verilator": {"mode": "lint-only", "exe": "false", "make_options": []},
         # verible, spyglass, ascentlint, slang...
-    ]
+    }
 
     FLOW_OPTIONS = {
         "frontends": {
@@ -29,23 +30,28 @@ class Lint(Edaflow):
         },
     }
 
-    def build_tool_graph(self):
+    @classmethod
+    def get_tool_options(cls, flow_options):
+        flow = flow_options.get("frontends", [])
+        tool = flow_options.get("tool")
+        if not tool:
+            raise RuntimeError("Flow 'lint' requires flow option 'tool' to be set")
+        flow.append(tool)
+
+        return cls.get_filtered_tool_options(flow, cls.FLOW_DEFINED_TOOL_OPTIONS)
+
+    def configure_flow(self, flow_options):
         tool = self.flow_options.get("tool", "")
         if not tool:
-            raise RuntimeError(
-                "'lint' backend requires flow option 'tool' to be defined"
-            )
-        tmp = [x for x in self.FLOW if x[0] == tool]
-        self.FLOW = tmp
-
-        # Note: This makes an assumption that the first tool in self.FLOW is
-        # a single entry point to the flow
-        next_tool = self.FLOW[0][0]
-
-        for frontend in reversed(self.flow_options.get("frontends", [])):
-            self.FLOW[0:0] = [(frontend, [next_tool], {})]
+            raise RuntimeError("Flow 'lint' requires flow option 'tool' to be set")
+        flow = [(tool, [], self.FLOW_DEFINED_TOOL_OPTIONS.get(tool, {}))]
+        # Add any user-specified frontends to the flow
+        next_tool = tool
+
+        for frontend in reversed(flow_options.get("frontends", [])):
+            flow[0:0] = [(frontend, [next_tool], {})]
             next_tool = frontend
-        return super().build_tool_graph()
+        return flow
 
     def configure_tools(self, nodes):
         super().configure_tools(nodes)

edalize/flows/sim.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
 import os.path
+from importlib import import_module
 
 from edalize.flows.edaflow import Edaflow
 
@@ -12,10 +13,7 @@ class Sim(Edaflow):
 
     argtypes = ["plusarg", "vlogdefine", "vlogparam"]
 
-    FLOW = [
-        ("verilator", [], {}),
-        # verible, spyglass, ascentlint, slang...
-    ]
+    FLOW_DEFINED_TOOL_OPTIONS = {}
 
     FLOW_OPTIONS = {
         "frontends": {
@@ -29,28 +27,28 @@ class Sim(Edaflow):
         },
     }
 
-    def build_tool_graph(self):
-        tool = self.flow_options.get("tool", "")
-
+    @classmethod
+    def get_tool_options(cls, flow_options):
+        flow = flow_options.get("frontends", [])
+        tool = flow_options.get("tool")
         if not tool:
             raise RuntimeError("Flow 'sim' requires flow option 'tool' to be set")
-        known_tools = [x[0] for x in self.FLOW]
-
-        self.FLOW = [x for x in self.FLOW if x[0] == tool]
+        flow.append(tool)
 
-        if not self.FLOW:
-            raise RuntimeError(
-                f"Unknown tool {tool!r}. Allowed options are {', '.join(known_tools)}"
-            )
+        return cls.get_filtered_tool_options(flow, cls.FLOW_DEFINED_TOOL_OPTIONS)
 
-        # FIXME: This makes an assumption that the first tool in self.FLOW is
-        # a single entry point to the flow
-        next_tool = self.FLOW[0][0]
+    def configure_flow(self, flow_options):
+        tool = self.flow_options.get("tool", "")
+        if not tool:
+            raise RuntimeError("Flow 'sim' requires flow option 'tool' to be set")
+        flow = [(tool, [], {})]
+        # Add any user-specified frontends to the flow
+        next_tool = tool
 
-        for frontend in reversed(self.flow_options.get("frontends", [])):
-            self.FLOW[0:0] = [(frontend, [next_tool], {})]
+        for frontend in reversed(flow_options.get("frontends", [])):
+            flow[0:0] = [(frontend, [next_tool], {})]
             next_tool = frontend
-        return super().build_tool_graph()
+        return flow
 
     def configure_tools(self, nodes):
         super().configure_tools(nodes)

edalize/flows/vivado.py

@@ -12,30 +12,53 @@ class Vivado(Edaflow):
 
     argtypes = ["vlogdefine", "vlogparam"]
 
-    FLOW = [
-        ("yosys", ["vivado"], {"arch": "xilinx", "output_format": "edif"}),
-        ("vivado", [], {}),
-    ]
+    FLOW_DEFINED_TOOL_OPTIONS = {
+        "yosys": {"arch": "xilinx", "output_format": "edif"},
+    }
 
     FLOW_OPTIONS = {
+        "frontends": {
+            "type": "str",
+            "desc": "Tools to run before yosys (e.g. sv2v)",
+            "list": True,
+        },
         "pnr": {
-            "type": "String",
+            "type": "str",
             "desc": "Select Place & Route tool.",
         },
         "synth": {
-            "type": "String",
+            "type": "str",
             "desc": "Synthesis tool. Allowed values are vivado (default) and yosys.",
         },
     }
 
-    def __init__(self, edam, work_root, verbose=False):
-        if edam.get("flow_options", {}).get("synth", {}) != "yosys":
-            # Remove from flow Yosys node
-            self.FLOW = [("vivado", [], {})]
-        Edaflow.__init__(self, edam, work_root, verbose)
+    @classmethod
+    def get_tool_options(cls, flow_options):
+        flow = flow_options.get("frontends", [])
+
+        if flow_options.get("synth") == "yosys":
+            flow.append("yosys")
+        flow.append("vivado")
+
+        return cls.get_filtered_tool_options(flow, cls.FLOW_DEFINED_TOOL_OPTIONS)
+
+    def configure_flow(self, flow_options):
+        flow = []
+        if flow_options.get("synth") == "yosys":
+            flow = [
+                ("yosys", ["vivado"], self.FLOW_DEFINED_TOOL_OPTIONS["yosys"]),
+                ("vivado", [], {"synth": "yosys"}),
+            ]
+            next_tool = "yosys"
+        else:
+            flow = [("vivado", [], {})]
+            next_tool = "vivado"
 
-    def build_tool_graph(self):
-        return super().build_tool_graph()
+        # Add any user-specified frontends to the flow
+        for frontend in reversed(flow_options.get("frontends", [])):
+            flow[0:0] = [(frontend, [next_tool], {})]
+            next_tool = frontend
+        return flow
 
     def configure_tools(self, nodes):
         super().configure_tools(nodes)