[ESI][Runtime] MMIO service support (#6034)

Service C++ interface, Python wrapping, and cosim support for MMIO. Also adds the basic service framework.

integration_test/Dialect/ESI/runtime/basic_mmio.sv

@@ -0,0 +1,94 @@
+// REQUIRES: esi-cosim, esi-runtime
+// RUN: esi-cosim-runner.py --exec %s.py %s
+
+// Test the low level cosim MMIO functionality. This test has 1024 64-bit
+// registers as a memory. It is an error to write to register 0.
+
+import Cosim_DpiPkg::*;
+
+module top(
+    input logic clk,
+    input logic rst
+);
+
+  // MMIO read: address channel.
+  logic        arvalid;
+  logic        arready;
+  logic [31:0] araddr;
+
+  // MMIO read: data response channel.
+  reg          rvalid;
+  logic        rready;
+  reg   [31:0] rdata;
+  reg   [1:0]  rresp;
+
+  // MMIO write: address channel.
+  logic        awvalid;
+  reg          awready;
+  logic [31:0] awaddr;
+
+  // MMIO write: data channel.
+  logic        wvalid;
+  reg          wready;
+  logic [31:0] wdata;
+
+  // MMIO write: write response channel.
+  reg          bvalid;
+  logic        bready;
+  reg   [1:0]  bresp;
+
+  Cosim_MMIO mmio (
+    .clk(clk),
+    .rst(rst),
+    .arvalid(arvalid),
+    .arready(arready),
+    .araddr(araddr),
+    .rvalid(rvalid),
+    .rready(rready),
+    .rdata(rdata),
+    .rresp(rresp),
+    .awvalid(awvalid),
+    .awready(awready),
+    .awaddr(awaddr),
+    .wvalid(wvalid),
+    .wready(wready),
+    .wdata(wdata),
+    .bvalid(bvalid),
+    .bready(bready),
+    .bresp(bresp)
+  );
+
+  reg [31:0] regs [1023:0];
+
+  assign arready = 1;
+  assign rdata = regs[araddr >> 3];
+  assign rresp = araddr == 0 ? 3 : 0;
+  always@(posedge clk) begin
+    if (rst) begin
+      rvalid <= 0;
+    end else begin
+      if (arvalid)
+        rvalid <= 1;
+      if (rready && rvalid)
+        rvalid <= 0;
+    end
+  end
+
+  wire write = awvalid && wvalid && !bvalid;
+  assign awready = write;
+  assign wready = write;
+  always@(posedge clk) begin
+    if (rst) begin
+      bvalid <= 0;
+    end else begin
+      if (bvalid && bready)
+        bvalid <= 0;
+      if (write) begin
+        bvalid <= 1;
+        bresp <= awaddr == 0 ? 3 : 0;
+        regs[awaddr >> 3] <= wdata;
+      end
+    end
+  end
+
+endmodule

integration_test/Dialect/ESI/runtime/basic_mmio.sv.py

@@ -0,0 +1,32 @@
+import esiaccel
+import os
+import sys
+
+conn = f"{sys.argv[1]}:{sys.argv[2]}"
+
+acc = esiaccel.Accelerator.connect("cosim", conn)
+mmio = acc.get_service_mmio()
+
+r = mmio.read(40)
+print(f"data resp: 0x{r:x}")
+
+try:
+  mmio.read(0)
+  assert False, "above should have thrown exception"
+except Exception:
+  print("caught expected exception")
+
+mmio.write(32, 86)
+r = mmio.read(32)
+print(f"data resp: 0x{r:x}")
+assert r == 86
+
+try:
+  mmio.write(0, 44)
+  assert False, "above should have thrown exception"
+except Exception:
+  print("caught expected exception")
+
+# Crashes with "magic num not found", which is expected since this isn't
+# supported yet.
+# acc.sysinfo().esi_version()

integration_test/Dialect/ESI/runtime/lit.local.cfg.py

@@ -0,0 +1,2 @@
+# Don't treat the Python files in this directory as tests.
+config.suffixes.remove('.py')

lib/Dialect/ESI/CMakeLists.txt

@@ -84,10 +84,12 @@ set(ESI_RUNTIME_SRCS
 
   runtime/CMakeLists.txt
   runtime/cpp/include/esi/Accelerator.h
+  runtime/cpp/include/esi/StdServices.h
   runtime/cpp/include/esi/backends/Cosim.h
   runtime/cpp/include/esi/backends/capnp.h
   runtime/cpp/include/esi/esi.h
   runtime/cpp/lib/Accelerator.cpp
+  runtime/cpp/lib/StdServices.cpp
   runtime/cpp/lib/backends/Cosim.cpp
   runtime/cpp/tools/esiquery.cpp
   runtime/python/esi/esiaccel.cpp

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -33,6 +33,7 @@ include_directories(cpp/include)
 
 set(ESIRuntimeSources
   cpp/lib/Accelerator.cpp
+  cpp/lib/StdServices.cpp
 )
 set(ESIRuntimeLinkLibraries)
 

lib/Dialect/ESI/runtime/cpp/include/esi/Accelerator.h

@@ -23,30 +23,56 @@
 
 #include <cstdint>
 #include <functional>
+#include <map>
 #include <memory>
 #include <string>
+#include <typeinfo>
 
 namespace esi {
-class SysInfo;
 
-/// An ESI accelerator system.
-class Accelerator {
-public:
-  virtual ~Accelerator() = default;
+constexpr uint32_t MagicNumOffset = 16;
+constexpr uint32_t MagicNumberLo = 0xE5100E51;
+constexpr uint32_t MagicNumberHi = 0x207D98E5;
+constexpr uint32_t VersionNumberOffset = MagicNumOffset + 8;
+constexpr uint32_t ExpectedVersionNumber = 0;
 
-  virtual const SysInfo &sysInfo() = 0;
+namespace services {
+/// Parent class of all APIs modeled as 'services'. May or may not map to a
+/// hardware side 'service'.
+class Service {
+public:
+  using Type = const std::type_info &;
+  virtual ~Service() = default;
 };
+} // namespace services
 
-/// Information about the Accelerator system.
-class SysInfo {
+/// An ESI accelerator system.
+class Accelerator {
 public:
-  virtual ~SysInfo() = default;
+  virtual ~Accelerator() = default;
 
-  /// Get the ESI version number to check version compatibility.
-  virtual uint32_t esiVersion() const = 0;
+  /// Get a typed reference to a particular service type. Caller does *not* take
+  /// ownership of the returned pointer -- the Accelerator object owns it.
+  /// Pointer lifetime ends with the Accelerator lifetime.
+  template <typename ServiceClass>
+  ServiceClass *getService() {
+    return dynamic_cast<ServiceClass *>(getServiceImpl(typeid(ServiceClass)));
+  }
 
-  /// Return the JSON-formatted system manifest.
-  virtual std::string rawJsonManifest() const = 0;
+protected:
+  using Service = services::Service;
+  /// Called by `getServiceImpl` exclusively. It wraps the pointer returned by
+  /// this in a unique_ptr and caches it. Separate this from the
+  /// wrapping/caching since wrapping/caching is an implementation detail.
+  virtual Service *createService(Service::Type service) = 0;
+  /// Calls `createService` and caches the result. Subclasses can override if
+  /// they want to use their own caching mechanism.
+  virtual Service *getServiceImpl(Service::Type service);
+
+private:
+  /// Cache services via a unique_ptr so they get free'd automatically when
+  /// Accelerator objects get deconstructed.
+  std::map<const std::type_info *, std::unique_ptr<Service>> serviceCache;
 };
 
 namespace registry {

lib/Dialect/ESI/runtime/cpp/include/esi/StdServices.h

@@ -0,0 +1,67 @@
+//===- StdServices.h - ESI standard services C++ API ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The APIs in this backend are all optionally implemented. The lower level
+// ones, however, are strongly recommended. 'Services' here refers to ESI
+// services. These are standard APIs into the standard ESI services.
+//
+// DO NOT EDIT!
+// This file is distributed as part of an ESI package. The source for this file
+// should always be modified within CIRCT.
+//
+//===----------------------------------------------------------------------===//
+
+// NOLINTNEXTLINE(llvm-header-guard)
+#ifndef ESI_RUNTIME_STDSERVICES_H
+#define ESI_RUNTIME_STDSERVICES_H
+
+#include "esi/Accelerator.h"
+
+#include <cstdint>
+
+namespace esi {
+namespace services {
+
+/// Information about the Accelerator system.
+class SysInfo : public Service {
+public:
+  virtual ~SysInfo() = default;
+
+  /// Get the ESI version number to check version compatibility.
+  virtual uint32_t esiVersion() const = 0;
+
+  /// Return the JSON-formatted system manifest.
+  virtual std::string rawJsonManifest() const = 0;
+};
+
+class MMIO : public Service {
+public:
+  virtual ~MMIO() = default;
+  virtual uint64_t read(uint32_t addr) const = 0;
+  virtual void write(uint32_t addr, uint64_t data) = 0;
+};
+
+/// Implement the SysInfo API for a standard MMIO protocol.
+class MMIOSysInfo final : public SysInfo {
+public:
+  MMIOSysInfo(const MMIO *);
+
+  /// Get the ESI version number to check version compatibility.
+  uint32_t esiVersion() const override;
+
+  /// Return the JSON-formatted system manifest.
+  std::string rawJsonManifest() const override;
+
+private:
+  const MMIO *mmio;
+};
+
+} // namespace services
+} // namespace esi
+
+#endif // ESI_RUNTIME_STDSERVICES_H

lib/Dialect/ESI/runtime/cpp/include/esi/backends/Cosim.h

@@ -22,22 +22,24 @@
 
 #include "esi/Accelerator.h"
 
+#include <memory>
+
 namespace esi {
 namespace backends {
 namespace cosim {
-class CosimSysInfo;
 
 /// Connect to an ESI simulation.
 class CosimAccelerator : public esi::Accelerator {
 public:
   CosimAccelerator(std::string hostname, uint16_t port);
-  ~CosimAccelerator();
   static std::unique_ptr<Accelerator> connect(std::string connectionString);
 
-  const SysInfo &sysInfo() override;
+protected:
+  virtual Service *createService(Service::Type service) override;
 
 private:
-  CosimSysInfo *info;
+  struct Impl;
+  std::unique_ptr<Impl> impl;
 };
 
 } // namespace cosim

lib/Dialect/ESI/runtime/cpp/lib/Accelerator.cpp

@@ -19,6 +19,12 @@
 #include <stdexcept>
 
 namespace esi {
+services::Service *Accelerator::getServiceImpl(Service::Type svcType) {
+  std::unique_ptr<Service> &cacheEntry = serviceCache[&svcType];
+  if (cacheEntry == nullptr)
+    cacheEntry = std::unique_ptr<Service>(createService(svcType));
+  return cacheEntry.get();
+}
 namespace registry {
 namespace internal {
 

lib/Dialect/ESI/runtime/cpp/lib/StdServices.cpp

@@ -0,0 +1,33 @@
+//===- StdServices.cpp - implementations of std services ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// DO NOT EDIT!
+// This file is distributed as part of an ESI package. The source for this file
+// should always be modified within CIRCT
+// (lib/dialect/ESI/runtime/cpp/lib/backends/Cosim.cpp).
+//
+//===----------------------------------------------------------------------===//
+
+#include "esi/StdServices.h"
+
+#include <stdexcept>
+
+using namespace esi;
+using namespace esi::services;
+
+MMIOSysInfo::MMIOSysInfo(const MMIO *mmio) : mmio(mmio) {}
+
+uint32_t MMIOSysInfo::esiVersion() const {
+  uint32_t hi = mmio->read(MagicNumOffset);
+  uint32_t lo = mmio->read(MagicNumOffset + 4);
+  if (hi != MagicNumberHi || lo != MagicNumberLo)
+    throw std::runtime_error("ESI magic number not found");
+  return mmio->read(VersionNumberOffset);
+}
+
+std::string MMIOSysInfo::rawJsonManifest() const { return ""; }