Add performance test for handle.py / BinaryValue

tests/test_cases/test_perf_handle/README.md

@@ -0,0 +1,6 @@
+# Cocotb Performance test
+
+Benchmarking different ways of reading / writing signals in cocotb
+
+When accessing signals, cocotb uses BinaryValue, this benchmark allows us to compare different ways
+of accessing signals.

tests/test_cases/test_perf_handle/hdl/dff8.sv

@@ -0,0 +1,62 @@
+// This file is public domain, it can be freely copied without restrictions.
+// SPDX-License-Identifier: CC0-1.0
+
+`timescale 1ns/1ps
+
+`ifdef VERILATOR  // make parameter readable from VPI
+  `define VL_RD /*verilator public_flat_rd*/
+`else
+  `define VL_RD
+`endif
+
+// D Flip Flop
+module dff8 #(
+    parameter int DATA_WIDTH `VL_RD = 8, // Data width in bits
+
+    // "Input Offset"
+    // Instead of 8 downto 0 use 11 downto 3
+    parameter int IOFF `VL_RD = 3,
+
+    // "Output Offset"
+    // Instead of 8 downto 0 use 13 downto 5
+    parameter int OOFF `VL_RD = 5
+) (
+    input clk_i,
+    input rst_i, // asynchronous reset
+
+    // Little endian: d_i[7] is the most significant bit
+    input [DATA_WIDTH-1+IOFF:0+IOFF] d_i, // Input
+    output logic [DATA_WIDTH-1+OOFF:0+OOFF] q_o, // Output
+    output logic [DATA_WIDTH-1+OOFF:0+OOFF] nq_o, // Negative output
+
+    // Big endian (BE): be_d_i[0] is the most significant bit
+    input [0+IOFF:DATA_WIDTH-1+IOFF] be_d_i, // Input
+    output logic [0+OOFF:DATA_WIDTH-1+OOFF] be_q_o, // Output
+    output logic [0+OOFF:DATA_WIDTH-1+OOFF] be_nq_o // Negative output
+    );
+
+    // Little Endian
+    logic [DATA_WIDTH-1:0] q_w;
+    always @(posedge clk_i, posedge rst_i) begin : proc_le_reg
+        if (rst_i == 1'b1) begin
+            q_w <= {DATA_WIDTH{1'b0}};
+        end else begin
+            q_w <= d_i;
+        end
+    end
+    assign nq_o = ~q_w;
+    assign q_o = q_w;
+
+    // Big Endian
+    logic [DATA_WIDTH-1:0] be_q_w;
+    always @(posedge clk_i, posedge rst_i) begin : proc_be_reg
+        if (rst_i == 1'b1) begin
+            be_q_w <= {DATA_WIDTH{1'b0}};
+        end else begin
+            be_q_w <= be_d_i;
+        end
+    end
+    assign be_nq_o = ~be_q_w;
+    assign be_q_o = be_q_w;
+
+endmodule

tests/test_cases/test_perf_handle/tests/Makefile

@@ -0,0 +1,51 @@
+# This file is public domain, it can be freely copied without restrictions.
+# SPDX-License-Identifier: CC0-1.0
+
+TOPLEVEL_LANG ?= verilog
+SIM ?= icarus
+
+PWD=$(shell pwd)
+
+# SystemVerilog parameters
+DATA_WIDTH ?= 8
+
+VERILOG_SOURCES = $(PWD)/../hdl/dff8.sv
+
+# Set module parameters
+ifeq ($(SIM),icarus)
+	COMPILE_ARGS += -Pdff8.DATA_WIDTH=$(DATA_WIDTH)
+else ifneq ($(filter $(SIM),questa modelsim riviera activehdl),)
+	SIM_ARGS += -gDATA_WIDTH=$(DATA_WIDTH)
+else ifeq ($(SIM),vcs)
+	COMPILE_ARGS += -pvalue+dff8/DATA_WIDTH=$(DATA_WIDTH)
+else ifeq ($(SIM),verilator)
+	COMPILE_ARGS += -GDATA_WIDTH=$(DATA_WIDTH)
+else ifneq ($(filter $(SIM),ius xcelium),)
+	EXTRA_ARGS += -defparam "dff8.DATA_WIDTH=$(DATA_WIDTH)"
+endif
+
+ifneq ($(filter $(SIM),riviera activehdl),)
+	COMPILE_ARGS += -sv2k12
+endif
+
+
+# Fix the seed to ensure deterministic tests
+export RANDOM_SEED := 123456789
+
+TOPLEVEL    := dff8
+MODULE      := test_dff8
+
+include $(shell cocotb-config --makefiles)/Makefile.sim
+
+# Profiling
+
+DOT_BINARY ?= dot
+
+test_profile.pstat: sim
+
+callgraph.svg: test_profile.pstat
+	$(shell cocotb-config --python-bin) -m gprof2dot -f pstats ./$< | $(DOT_BINARY) -Tsvg -o $@
+
+.PHONY: profile
+profile:
+	COCOTB_ENABLE_PROFILING=1 $(MAKE) callgraph.svg

tests/test_cases/test_perf_handle/tests/test_dff8.py

@@ -0,0 +1,272 @@
+# This file is public domain, it can be freely copied without restrictions.
+# SPDX-License-Identifier: CC0-1.0
+
+import os
+import sys
+import time
+from pathlib import Path
+
+import cocotb
+from cocotb.clock import Clock
+from cocotb.runner import get_runner
+from cocotb.triggers import RisingEdge
+
+NUM_SAMPLES = int(os.environ.get("NUM_SAMPLES", 10000))
+if cocotb.simulator.is_running():
+    DATA_WIDTH = int(cocotb.top.DATA_WIDTH)
+
+
+def test_dff8_runner():
+    """
+    Simulate the "dff8" example
+    This file can be run directly or via pytest discovery.
+    """
+    hdl_toplevel_lang = os.getenv("HDL_TOPLEVEL_LANG", "verilog")
+    sim = os.getenv("SIM", "icarus")
+
+    proj_path = Path(__file__).resolve().parent.parent
+
+    verilog_sources = []
+    vhdl_sources = []
+    build_args = []
+
+    if hdl_toplevel_lang == "verilog":
+        verilog_sources = [proj_path / "hdl" / "dff8.sv"]
+        if sim in ["riviera", "activehdl"]:
+            build_args = ["-sv2k12"]
+    else:
+        raise ValueError(
+            f"A valid value verilog file was not provided for TOPLEVEL_LANG={hdl_toplevel_lang}"
+        )
+
+    parameters = {
+        "DATA_WIDTH": "8",
+    }
+
+    # equivalent to setting the PYTHONPATH environment variable
+    sys.path.append(str(proj_path / "tests"))
+
+    runner = get_runner(sim)
+    runner.build(
+        hdl_toplevel="dff8",
+        verilog_sources=verilog_sources,
+        vhdl_sources=vhdl_sources,
+        build_args=build_args,
+        parameters=parameters,
+        always=True,
+    )
+    runner.test(
+        hdl_toplevel="dff8",
+        hdl_toplevel_lang=hdl_toplevel_lang,
+        test_module="test_dff8",
+    )
+
+
+@cocotb.test()
+async def test_dff8(dut):
+    """
+    Cocotb performance test. Measure how long it takes to access DUT's signals
+    with different methods.
+
+    The environment variable NUM_SAMPLES can be used to configure how many
+    iterations are performed.
+
+    DUT's signals:
+        - clk_i
+        - rst_i
+
+        - d_i, 8-bit
+        - q_o, 8-bit
+        - nq_o, 8-bit
+
+        - be_d_i, 8-bit
+        - be_q_o, 8-bit
+        - be_nq_o, 8-bit
+
+    This test does not explicitly call BinaryValue or LogicArray. It
+    can be used to compare multiple implementation together.
+    """
+    cocotb.start_soon(Clock(dut.clk_i, 10, units="us").start(start_high=False))
+
+    dut._log.info("Initialize and reset model")
+
+    dut.d_i.value = 0
+    dut.be_d_i.value = 0
+
+    # Reset sequence
+    dut.rst_i.value = 1
+    for _ in range(3):
+        await RisingEdge(dut.clk_i)
+    dut.rst_i.value = 0
+
+    # Little Endian, setting input signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        dut.d_i.value = i % 256
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Little Endian, setting input signal: {elapsed_time}s")
+    await RisingEdge(dut.clk_i)
+
+    # Big Endian, setting input signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        dut.be_d_i.value = i % 256
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Big Endian, setting input signal: {elapsed_time}s")
+    await RisingEdge(dut.clk_i)
+
+    # Little Endian, reading output signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        _ = dut.q_o.value
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Little Endian, reading the output signal: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+    # Big Endian, reading output signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        _ = dut.be_q_o.value
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Big Endian, reading the output signal: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+    # Little Endian, reading part of the output signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        _ = dut.q_o.value[0:6]  # BinaryValue requires indexes to be from low to high
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Little Endian, reading part of the output signal: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+    # Big Endian, reading part of the output signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        _ = dut.be_q_o.value[0:6]  # BinaryValue requires indexes to be from low to high
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Big Endian Endian, reading part of the output sign: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+    # Little Endian, reading single bit from the output signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        _ = dut.q_o.value[4]
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(
+        f"Little Endian, reading single bit from the output signal: {elapsed_time}"
+    )
+    await RisingEdge(dut.clk_i)
+
+    # Big Endian, reading single bit from the output signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        _ = dut.be_q_o.value[4]
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(
+        f"Big Endian, reading single bit from the output signal: {elapsed_time}"
+    )
+    await RisingEdge(dut.clk_i)
+
+    # Little Endian, setting part of the input signal
+    # handle.py: Slice indexing is not supported
+    await RisingEdge(dut.clk_i)
+    raised_error = ""
+    try:
+        dut.d_i[6:0].value = 42
+    except IndexError as e:
+        raised_error = str(e)
+    assert raised_error == "Slice indexing is not supported"
+    dut._log.info("Little Endian, setting part of the input signal: Unsupported")
+
+    # Big Endian, setting part of the input signal
+    # handle.py: Slice indexing is not supported
+    await RisingEdge(dut.clk_i)
+    raised_error = ""
+    try:
+        dut.be_d_i[6:0].value = 42
+    except IndexError as e:
+        raised_error = str(e)
+    assert raised_error == "Slice indexing is not supported"
+    dut._log.info("Big Endian, setting part of the input signal: Unsupported")
+
+    # Little Endian, setting single bit of the input signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        dut.d_i[4].value = i % 2
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(
+        f"Little Endian, setting single bit of the input signal: {elapsed_time}"
+    )
+    await RisingEdge(dut.clk_i)
+
+    # Big Endian, setting single bit of the input signal
+    start_time = time.perf_counter()
+    for i in range(NUM_SAMPLES):
+        await RisingEdge(dut.clk_i)
+        dut.be_d_i[4].value = i % 2
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Big Endian, setting single bit of the input signal: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+    # Little Endian, read + write signals
+    start_time = time.perf_counter()
+    dut.d_i.value = 0
+    await RisingEdge(dut.clk_i)
+    expected = 0
+    for i in range(NUM_SAMPLES):
+        # Set input signals
+        val = i % 256
+        dut.d_i.value = val
+        await RisingEdge(dut.clk_i)
+        # Check output signals on next cycle
+        assert (
+            dut.q_o.value == expected
+        ), "signal Q should take the value of D after 1 cycle"
+        expected = val
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Little Endian, read + write signals: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+    # Big Endian, read + write signals
+    start_time = time.perf_counter()
+    dut.be_d_i.value = 0
+    await RisingEdge(dut.clk_i)
+    expected = 0
+    for i in range(NUM_SAMPLES):
+        # Set input signals
+        val = i % 256
+        dut.be_d_i.value = val
+        await RisingEdge(dut.clk_i)
+        # Check output signals on next cycle
+        assert (
+            dut.be_q_o.value == expected
+        ), "signal Q should take the value of D after 1 cycle"
+        expected = val
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    dut._log.info(f"Big Endian, read + write signals: {elapsed_time}")
+    await RisingEdge(dut.clk_i)
+
+
+if __name__ == "__main__":
+    test_dff8_runner()