diff --git a/.gitignore b/.gitignore index 5161d4101..48ff3718f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,11 @@ BlinkyExample/ Pipfile.lock +# Python packaging +build +edg.egg-info +dist + # mypy .mypy_cache/ .dmypy.json diff --git a/LibraryDump.py b/LibraryDump.py deleted file mode 100644 index ad4826eb7..000000000 --- a/LibraryDump.py +++ /dev/null @@ -1,45 +0,0 @@ -# Simple tool that scans for libraries and dumps the whole thing to a proto file -from typing import cast - -import edgir -from HdlInterfaceService import LibraryElementIndexer -import edg_core -import edg -from edg_core.Builder import builder - - -OUTPUT_FILE = "library.edg" - -if __name__ == '__main__': - library = LibraryElementIndexer() - indexed = library.index_module(edg) - pb = edgir.Library() - - count = 0 - for cls in indexed: - name = cls._static_def_name() - obj = cls() - if isinstance(obj, edg_core.Block): - print(f"Elaborating block {name}") - block_proto = builder.elaborate_toplevel(obj) - pb.root.members[name].hierarchy_block.CopyFrom(block_proto) - elif isinstance(obj, edg_core.Link): - print(f"Elaborating link {name}") - link_proto = builder.elaborate_toplevel(obj) - assert isinstance(link_proto, edgir.Link) # TODO this needs to be cleaned up - pb.root.members[name].link.CopyFrom(link_proto) - elif isinstance(obj, edg_core.Bundle): # TODO: note Bundle extends Port, so this must come first - print(f"Elaborating bundle {name}") - pb.root.members[name].bundle.CopyFrom(obj._def_to_proto()) - elif isinstance(obj, edg_core.Port): - print(f"Elaborating port {name}") - pb.root.members[name].port.CopyFrom(cast(edgir.Port, obj._def_to_proto())) - else: - print(f"Unknown category for class {cls}") - - count += 1 - - with open(OUTPUT_FILE, 'wb') as file: - file.write(pb.SerializeToString()) - - print(f"Wrote {count} classes to {OUTPUT_FILE}") diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..86a0af514 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include edg/edg-compiler-precompiled.jar diff --git a/compiler/src/main/scala/edg/compiler/CompilerServerMain.scala b/compiler/src/main/scala/edg/compiler/CompilerServerMain.scala index f2ef93ac0..82285f16d 100644 --- a/compiler/src/main/scala/edg/compiler/CompilerServerMain.scala +++ b/compiler/src/main/scala/edg/compiler/CompilerServerMain.scala @@ -14,8 +14,8 @@ import java.io.{File, PrintWriter, StringWriter} // a PythonInterface that uses the on-event hooks to forward stderr and stdout // without this, the compiler can freeze on large stdout/stderr data, possibly because of queue sizing -class ForwardingPythonInterface(serverFile: File) - extends PythonInterface(serverFile) { +class ForwardingPythonInterface(serverFile: Option[File], pythonPaths: Seq[String]) + extends PythonInterface(serverFile, pythonPaths) { def forwardProcessOutput(): Unit = { StreamUtils.forAvailable(processOutputStream) { data => System.out.print(new String(data)) @@ -79,7 +79,9 @@ object CompilerServerMain { } def main(args: Array[String]): Unit = { - val pyIf = new ForwardingPythonInterface(new File("HdlInterfaceService.py")) // use relative path + val hdlServerOption = PythonInterface.serverFileOption(None) // local relative path + hdlServerOption.foreach { serverFile => println(s"Using local $serverFile") } + val pyIf = new ForwardingPythonInterface(hdlServerOption, Seq(new File(".").getAbsolutePath)) val pyLib = new PythonInterfaceLibrary() pyLib.withPythonInterface(pyIf) { while (true) { diff --git a/compiler/src/main/scala/edg/compiler/PythonInterface.scala b/compiler/src/main/scala/edg/compiler/PythonInterface.scala index 4a2b89fb9..0759c1665 100644 --- a/compiler/src/main/scala/edg/compiler/PythonInterface.scala +++ b/compiler/src/main/scala/edg/compiler/PythonInterface.scala @@ -21,12 +21,20 @@ object ProtobufStdioSubprocess { } -class ProtobufStdioSubprocess - [RequestType <: scalapb.GeneratedMessage, ResponseType <: scalapb.GeneratedMessage]( +class ProtobufStdioSubprocess[RequestType <: scalapb.GeneratedMessage, ResponseType <: scalapb.GeneratedMessage]( responseType: scalapb.GeneratedMessageCompanion[ResponseType], - args: Seq[String]) { + pythonPaths: Seq[String], args: Seq[String]) { protected val process: Either[Process, Throwable] = try { - Left(new ProcessBuilder(args: _*).start()) + val processBuilder = new ProcessBuilder(args: _*) + if (pythonPaths.nonEmpty) { + val env = processBuilder.environment() + val pythonPathString = pythonPaths.mkString(";") + Option(env.get("PYTHONPATH")) match { // merge existing PYTHONPATH if exists + case None => env.put("PYTHONPATH", pythonPathString) + case Some(envPythonPath) => env.put("PYTHONPATH", envPythonPath + ";" + pythonPathString) + } + } + Left(processBuilder.start()) } catch { case e: Throwable => Right(e) // if it fails store the exception to be thrown when we can } @@ -113,14 +121,36 @@ class ProtobufStdioSubprocess } +object PythonInterface { + private val kHdlServerFilePath = "edg_hdl_server/__main__.py" + // returns the HDL server Python script if it exists locally, otherwise returns None. + def serverFileOption(root: Option[File] = None): Option[File] = { + val hdlServerFile = root match { + case Some(root) => new File(root, kHdlServerFilePath) + case None => new File(kHdlServerFilePath) + } + if (hdlServerFile.exists()) { + Some(hdlServerFile) + } else { + None + } + } +} + + /** An interface to the Python HDL elaborator, which reads in Python HDL code and (partially) compiles * them down to IR. * The underlying Python HDL should not change while this is open. This will not reload updated Python HDL files. + * + * If the serverFile is specified, run that; otherwise use "python -m edg_hdl_server" for the global package. */ -class PythonInterface(serverFile: File, pythonInterpreter: String = "python") { +class PythonInterface(serverFile: Option[File], pythonPaths: Seq[String], pythonInterpreter: String = "python") { + val command = serverFile match { // -u for unbuffered mode + case Some(serverFile) => Seq(pythonInterpreter, "-u", serverFile.getAbsolutePath) + case None => Seq(pythonInterpreter, "-u", "-m", "edg_hdl_server") + } protected val process = new ProtobufStdioSubprocess[edgrpc.HdlRequest, edgrpc.HdlResponse]( - edgrpc.HdlResponse, - Seq(pythonInterpreter, "-u", serverFile.getAbsolutePath)) // in unbuffered mode + edgrpc.HdlResponse, pythonPaths, command) val processOutputStream: InputStream = process.outputStream val processErrorStream: InputStream = process.errorStream diff --git a/compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala b/compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala index ccd982a61..8ace3aae7 100644 --- a/compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala +++ b/compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala @@ -13,7 +13,8 @@ class PythonInterfaceTest extends AnyFlatSpec { val compiledDir = new File(getClass.getResource("").getPath) // above returns compiler/target/scala-2.xx/test-classes/edg/compiler, get the root repo dir val repoDir = compiledDir.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile - val pyIf = new PythonInterface(new File(repoDir, "HdlInterfaceService.py")) + val pyIf = new PythonInterface(Some(new File(repoDir, "edg_hdl_server/__main__.py")), + Seq(repoDir.getAbsolutePath)) pyIf.indexModule("edg_core").getClass should equal(classOf[Errorable.Success[Seq[LibraryPath]]]) pyIf.shutdown() should equal(0) } diff --git a/edg_core/ScalaCompilerInterface.py b/edg_core/ScalaCompilerInterface.py index decd61982..27beba5c5 100644 --- a/edg_core/ScalaCompilerInterface.py +++ b/edg_core/ScalaCompilerInterface.py @@ -50,8 +50,8 @@ def append_values(self, values: List[Tuple[edgir.LocalPath, edgir.ValueLit]]): class ScalaCompilerInstance: - PRECOMPIED_RELPATH = "compiler/edg-compiler-precompiled.jar" - DEV_RELPATH = "compiler/target/scala-2.13/edg-compiler-assembly-0.1-SNAPSHOT.jar" + DEV_RELPATH = "../compiler/target/scala-2.13/edg-compiler-assembly-0.1-SNAPSHOT.jar" + PRECOMPIED_RELPATH = "resources/edg-compiler-precompiled.jar" def __init__(self, *, suppress_stderr: bool = False): self.process: Optional[Any] = None @@ -61,13 +61,15 @@ def __init__(self, *, suppress_stderr: bool = False): def check_started(self) -> None: if self.process is None: - if os.path.exists(self.DEV_RELPATH): - jar_path = self.DEV_RELPATH + dev_path = os.path.join(os.path.dirname(__file__), self.DEV_RELPATH) + precompiled_path = os.path.join(os.path.dirname(__file__), self.PRECOMPIED_RELPATH) + if os.path.exists(dev_path): + jar_path = dev_path print("Using development JAR") - elif os.path.exists(self.PRECOMPIED_RELPATH): - jar_path = self.PRECOMPIED_RELPATH + elif os.path.exists(precompiled_path): + jar_path = precompiled_path else: - raise ValueError("No EDG Compiler JAR found") + raise ValueError(f"No EDG Compiler JAR found") self.process = subprocess.Popen( ['java', '-jar', jar_path], diff --git a/compiler/edg-compiler-precompiled.jar b/edg_core/resources/edg-compiler-precompiled.jar similarity index 96% rename from compiler/edg-compiler-precompiled.jar rename to edg_core/resources/edg-compiler-precompiled.jar index 979739827..722ec4aab 100644 Binary files a/compiler/edg-compiler-precompiled.jar and b/edg_core/resources/edg-compiler-precompiled.jar differ diff --git a/edg_hdl_server/__init__.py b/edg_hdl_server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/HdlInterfaceService.py b/edg_hdl_server/__main__.py similarity index 93% rename from HdlInterfaceService.py rename to edg_hdl_server/__main__.py index fc4a3a7c7..3ccad8c55 100644 --- a/HdlInterfaceService.py +++ b/edg_hdl_server/__main__.py @@ -10,6 +10,9 @@ from edg_core.Core import NonLibraryProperty +EDG_PROTO_VERSION = 1 + + class LibraryElementIndexer: """Indexer for libraries, recursively searches modules and their LibraryElements.""" def __init__(self): @@ -24,8 +27,8 @@ def index_module(self, module: ModuleType) -> Set[Type[LibraryElement]]: def _search_module(self, module: ModuleType) -> None: # avoid repeated work and re-indexing modules if (module.__name__ in sys.builtin_module_names - or not hasattr(module, '__file__') # apparently load six.moves breaks - or module in self.seen_modules): + or not hasattr(module, '__file__') # apparently load six.moves breaks + or module in self.seen_modules): return self.seen_modules.add(module) @@ -34,8 +37,8 @@ def _search_module(self, module: ModuleType) -> None: self._search_module(member) if inspect.isclass(member) and issubclass(member, LibraryElement) and not issubclass(member, DesignTop) \ - and member not in self.seen_elements \ - and (member, NonLibraryProperty) not in member._elt_properties: # process elements + and member not in self.seen_elements \ + and (member, NonLibraryProperty) not in member._elt_properties: # process elements self.seen_elements.add(member) for mro in member.mro(): @@ -67,7 +70,7 @@ def elaborate_class(elt_cls: Type[LibraryElementType]) -> Tuple[LibraryElementTy LibraryClassType = TypeVar('LibraryClassType') def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[LibraryClassType]) -> \ - Type[LibraryClassType]: + Type[LibraryClassType]: elt_split = elt.target.name.split('.') elt_module = importlib.import_module('.'.join(elt_split[:-1])) assert inspect.ismodule(elt_module) @@ -76,9 +79,7 @@ def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[Library return cls -# In some cases stdout seems to buffer excessively, in which case starting python with -u seems to work -# https://stackoverflow.com/a/35467658/5875811 -if __name__ == '__main__': +def run_server(): stdin_deserializer = BufferDeserializer(edgrpc.HdlRequest, sys.stdin.buffer) stdout_serializer = BufferSerializer[edgrpc.HdlResponse](sys.stdout.buffer) @@ -137,6 +138,8 @@ def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[Library response_result = response.run_backend.results.add() response_result.path.CopyFrom(path) response_result.text = backend_result + elif request.HasField('get_proto_version'): + response.get_proto_version = EDG_PROTO_VERSION else: raise RuntimeError(f"Unknown request {request}") except BaseException as e: @@ -149,3 +152,9 @@ def class_from_library(elt: edgir.LibraryPath, expected_superclass: Type[Library sys.stdout.buffer.write(stdin_deserializer.read_stdout()) stdout_serializer.write(response) + + +# In some cases stdout seems to buffer excessively, in which case starting python with -u seems to work +# https://stackoverflow.com/a/35467658/5875811 +if __name__ == '__main__': + run_server() diff --git a/edgrpc/hdl_pb2.py b/edgrpc/hdl_pb2.py index ac8d9237a..4edfcd41b 100644 --- a/edgrpc/hdl_pb2.py +++ b/edgrpc/hdl_pb2.py @@ -17,7 +17,7 @@ from edgir import lit_pb2 as edgir_dot_lit__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x65\x64grpc/hdl.proto\x12\nedgrpc.hdl\x1a\x12\x65\x64gir/schema.proto\x1a\x0f\x65\x64gir/ref.proto\x1a\x10\x65\x64gir/elem.proto\x1a\x0f\x65\x64gir/lit.proto\"\xb6\x04\n\x0bRefinements\x12\x34\n\nsubclasses\x18\x01 \x03(\x0b\x32 .edgrpc.hdl.Refinements.Subclass\x12-\n\x06values\x18\x02 \x03(\x0b\x32\x1d.edgrpc.hdl.Refinements.Value\x1a\x8e\x01\n\x08Subclass\x12$\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPathH\x00\x12%\n\x03\x63ls\x18\x02 \x01(\x0b\x32\x16.edgir.ref.LibraryPathH\x00\x12+\n\x0breplacement\x18\x03 \x01(\x0b\x32\x16.edgir.ref.LibraryPathB\x08\n\x06source\x1a\xb0\x02\n\x05Value\x12$\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPathH\x00\x12\x41\n\tcls_param\x18\x02 \x01(\x0b\x32,.edgrpc.hdl.Refinements.Value.ClassParamPathH\x00\x12#\n\x04\x65xpr\x18\x03 \x01(\x0b\x32\x13.edgir.lit.ValueLitH\x01\x12%\n\x05param\x18\x04 \x01(\x0b\x32\x14.edgir.ref.LocalPathH\x01\x1a_\n\x0e\x43lassParamPath\x12#\n\x03\x63ls\x18\x01 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12(\n\nparam_path\x18\x02 \x01(\x0b\x32\x14.edgir.ref.LocalPathB\x08\n\x06sourceB\x07\n\x05value\"\x1a\n\nModuleName\x12\x0c\n\x04name\x18\x01 \x01(\t\"8\n\rIndexResponse\x12\'\n\x07indexed\x18\x01 \x03(\x0b\x32\x16.edgir.ref.LibraryPath\"9\n\x0eLibraryRequest\x12\'\n\x07\x65lement\x18\x02 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\"n\n\x0fLibraryResponse\x12-\n\x07\x65lement\x18\x01 \x01(\x0b\x32\x1c.edgir.schema.Library.NS.Val\x12,\n\x0brefinements\x18\x03 \x01(\x0b\x32\x17.edgrpc.hdl.Refinements\"S\n\tExprValue\x12\"\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPath\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.edgir.lit.ValueLit\"b\n\x10GeneratorRequest\x12\'\n\x07\x65lement\x18\x02 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12%\n\x06values\x18\x04 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\"B\n\x11GeneratorResponse\x12-\n\tgenerated\x18\x01 \x01(\x0b\x32\x1a.edgir.elem.HierarchyBlock\"\x97\x01\n\x11RefinementRequest\x12/\n\x0frefinement_pass\x18\x01 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12$\n\x06\x64\x65sign\x18\x02 \x01(\x0b\x32\x14.edgir.schema.Design\x12+\n\x0csolvedValues\x18\x03 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\">\n\x12RefinementResponse\x12(\n\tnewValues\x18\x01 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\"\xfc\x01\n\x0e\x42\x61\x63kendRequest\x12\'\n\x07\x62\x61\x63kend\x18\x01 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12$\n\x06\x64\x65sign\x18\x02 \x01(\x0b\x32\x14.edgir.schema.Design\x12+\n\x0csolvedValues\x18\x03 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\x12<\n\targuments\x18\x04 \x03(\x0b\x32).edgrpc.hdl.BackendRequest.ArgumentsEntry\x1a\x30\n\x0e\x41rgumentsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8e\x01\n\x0f\x42\x61\x63kendResponse\x12\x33\n\x07results\x18\x01 \x03(\x0b\x32\".edgrpc.hdl.BackendResponse.Result\x1a\x46\n\x06Result\x12\"\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPath\x12\x0e\n\x04text\x18\x02 \x01(\tH\x00\x42\x08\n\x06result\"1\n\rErrorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x11\n\ttraceback\x18\x02 \x01(\t\"\xab\x02\n\nHdlRequest\x12.\n\x0cindex_module\x18\x01 \x01(\x0b\x32\x16.edgrpc.hdl.ModuleNameH\x00\x12\x39\n\x13get_library_element\x18\x02 \x01(\x0b\x32\x1a.edgrpc.hdl.LibraryRequestH\x00\x12;\n\x13\x65laborate_generator\x18\x03 \x01(\x0b\x32\x1c.edgrpc.hdl.GeneratorRequestH\x00\x12\x37\n\x0erun_refinement\x18\x05 \x01(\x0b\x32\x1d.edgrpc.hdl.RefinementRequestH\x00\x12\x31\n\x0brun_backend\x18\x04 \x01(\x0b\x32\x1a.edgrpc.hdl.BackendRequestH\x00\x42\t\n\x07request\"\xe0\x02\n\x0bHdlResponse\x12\x31\n\x0cindex_module\x18\x01 \x01(\x0b\x32\x19.edgrpc.hdl.IndexResponseH\x00\x12:\n\x13get_library_element\x18\x02 \x01(\x0b\x32\x1b.edgrpc.hdl.LibraryResponseH\x00\x12<\n\x13\x65laborate_generator\x18\x03 \x01(\x0b\x32\x1d.edgrpc.hdl.GeneratorResponseH\x00\x12\x38\n\x0erun_refinement\x18\x05 \x01(\x0b\x32\x1e.edgrpc.hdl.RefinementResponseH\x00\x12\x32\n\x0brun_backend\x18\x04 \x01(\x0b\x32\x1b.edgrpc.hdl.BackendResponseH\x00\x12*\n\x05\x65rror\x18\x63 \x01(\x0b\x32\x19.edgrpc.hdl.ErrorResponseH\x00\x42\n\n\x08responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x65\x64grpc/hdl.proto\x12\nedgrpc.hdl\x1a\x12\x65\x64gir/schema.proto\x1a\x0f\x65\x64gir/ref.proto\x1a\x10\x65\x64gir/elem.proto\x1a\x0f\x65\x64gir/lit.proto\"\xb6\x04\n\x0bRefinements\x12\x34\n\nsubclasses\x18\x01 \x03(\x0b\x32 .edgrpc.hdl.Refinements.Subclass\x12-\n\x06values\x18\x02 \x03(\x0b\x32\x1d.edgrpc.hdl.Refinements.Value\x1a\x8e\x01\n\x08Subclass\x12$\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPathH\x00\x12%\n\x03\x63ls\x18\x02 \x01(\x0b\x32\x16.edgir.ref.LibraryPathH\x00\x12+\n\x0breplacement\x18\x03 \x01(\x0b\x32\x16.edgir.ref.LibraryPathB\x08\n\x06source\x1a\xb0\x02\n\x05Value\x12$\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPathH\x00\x12\x41\n\tcls_param\x18\x02 \x01(\x0b\x32,.edgrpc.hdl.Refinements.Value.ClassParamPathH\x00\x12#\n\x04\x65xpr\x18\x03 \x01(\x0b\x32\x13.edgir.lit.ValueLitH\x01\x12%\n\x05param\x18\x04 \x01(\x0b\x32\x14.edgir.ref.LocalPathH\x01\x1a_\n\x0e\x43lassParamPath\x12#\n\x03\x63ls\x18\x01 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12(\n\nparam_path\x18\x02 \x01(\x0b\x32\x14.edgir.ref.LocalPathB\x08\n\x06sourceB\x07\n\x05value\"\x1a\n\nModuleName\x12\x0c\n\x04name\x18\x01 \x01(\t\"8\n\rIndexResponse\x12\'\n\x07indexed\x18\x01 \x03(\x0b\x32\x16.edgir.ref.LibraryPath\"9\n\x0eLibraryRequest\x12\'\n\x07\x65lement\x18\x02 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\"n\n\x0fLibraryResponse\x12-\n\x07\x65lement\x18\x01 \x01(\x0b\x32\x1c.edgir.schema.Library.NS.Val\x12,\n\x0brefinements\x18\x03 \x01(\x0b\x32\x17.edgrpc.hdl.Refinements\"S\n\tExprValue\x12\"\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPath\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.edgir.lit.ValueLit\"b\n\x10GeneratorRequest\x12\'\n\x07\x65lement\x18\x02 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12%\n\x06values\x18\x04 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\"B\n\x11GeneratorResponse\x12-\n\tgenerated\x18\x01 \x01(\x0b\x32\x1a.edgir.elem.HierarchyBlock\"\x97\x01\n\x11RefinementRequest\x12/\n\x0frefinement_pass\x18\x01 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12$\n\x06\x64\x65sign\x18\x02 \x01(\x0b\x32\x14.edgir.schema.Design\x12+\n\x0csolvedValues\x18\x03 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\">\n\x12RefinementResponse\x12(\n\tnewValues\x18\x01 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\"\xfc\x01\n\x0e\x42\x61\x63kendRequest\x12\'\n\x07\x62\x61\x63kend\x18\x01 \x01(\x0b\x32\x16.edgir.ref.LibraryPath\x12$\n\x06\x64\x65sign\x18\x02 \x01(\x0b\x32\x14.edgir.schema.Design\x12+\n\x0csolvedValues\x18\x03 \x03(\x0b\x32\x15.edgrpc.hdl.ExprValue\x12<\n\targuments\x18\x04 \x03(\x0b\x32).edgrpc.hdl.BackendRequest.ArgumentsEntry\x1a\x30\n\x0e\x41rgumentsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8e\x01\n\x0f\x42\x61\x63kendResponse\x12\x33\n\x07results\x18\x01 \x03(\x0b\x32\".edgrpc.hdl.BackendResponse.Result\x1a\x46\n\x06Result\x12\"\n\x04path\x18\x01 \x01(\x0b\x32\x14.edgir.ref.LocalPath\x12\x0e\n\x04text\x18\x02 \x01(\tH\x00\x42\x08\n\x06result\"1\n\rErrorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x11\n\ttraceback\x18\x02 \x01(\t\"\xc8\x02\n\nHdlRequest\x12.\n\x0cindex_module\x18\x01 \x01(\x0b\x32\x16.edgrpc.hdl.ModuleNameH\x00\x12\x39\n\x13get_library_element\x18\x02 \x01(\x0b\x32\x1a.edgrpc.hdl.LibraryRequestH\x00\x12;\n\x13\x65laborate_generator\x18\x03 \x01(\x0b\x32\x1c.edgrpc.hdl.GeneratorRequestH\x00\x12\x37\n\x0erun_refinement\x18\x05 \x01(\x0b\x32\x1d.edgrpc.hdl.RefinementRequestH\x00\x12\x31\n\x0brun_backend\x18\x04 \x01(\x0b\x32\x1a.edgrpc.hdl.BackendRequestH\x00\x12\x1b\n\x11get_proto_version\x18Z \x01(\rH\x00\x42\t\n\x07request\"\xfd\x02\n\x0bHdlResponse\x12\x31\n\x0cindex_module\x18\x01 \x01(\x0b\x32\x19.edgrpc.hdl.IndexResponseH\x00\x12:\n\x13get_library_element\x18\x02 \x01(\x0b\x32\x1b.edgrpc.hdl.LibraryResponseH\x00\x12<\n\x13\x65laborate_generator\x18\x03 \x01(\x0b\x32\x1d.edgrpc.hdl.GeneratorResponseH\x00\x12\x38\n\x0erun_refinement\x18\x05 \x01(\x0b\x32\x1e.edgrpc.hdl.RefinementResponseH\x00\x12\x32\n\x0brun_backend\x18\x04 \x01(\x0b\x32\x1b.edgrpc.hdl.BackendResponseH\x00\x12\x1b\n\x11get_proto_version\x18Z \x01(\rH\x00\x12*\n\x05\x65rror\x18\x63 \x01(\x0b\x32\x19.edgrpc.hdl.ErrorResponseH\x00\x42\n\n\x08responseb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'edgrpc.hdl_pb2', globals()) @@ -63,7 +63,7 @@ _ERRORRESPONSE._serialized_start=1801 _ERRORRESPONSE._serialized_end=1850 _HDLREQUEST._serialized_start=1853 - _HDLREQUEST._serialized_end=2152 - _HDLRESPONSE._serialized_start=2155 - _HDLRESPONSE._serialized_end=2507 + _HDLREQUEST._serialized_end=2181 + _HDLRESPONSE._serialized_start=2184 + _HDLRESPONSE._serialized_end=2565 # @@protoc_insertion_point(module_scope) diff --git a/edgrpc/hdl_pb2.pyi b/edgrpc/hdl_pb2.pyi index 2dfd83151..32a5ef58c 100644 --- a/edgrpc/hdl_pb2.pyi +++ b/edgrpc/hdl_pb2.pyi @@ -410,6 +410,7 @@ class HdlRequest(google.protobuf.message.Message): ELABORATE_GENERATOR_FIELD_NUMBER: builtins.int RUN_REFINEMENT_FIELD_NUMBER: builtins.int RUN_BACKEND_FIELD_NUMBER: builtins.int + GET_PROTO_VERSION_FIELD_NUMBER: builtins.int @property def index_module(self) -> global___ModuleName: """returns an index of IR elements in a Python module""" @@ -423,6 +424,8 @@ class HdlRequest(google.protobuf.message.Message): def run_refinement(self) -> global___RefinementRequest: ... @property def run_backend(self) -> global___BackendRequest: ... + get_proto_version: builtins.int + """no data""" def __init__( self, *, @@ -431,10 +434,11 @@ class HdlRequest(google.protobuf.message.Message): elaborate_generator: global___GeneratorRequest | None = ..., run_refinement: global___RefinementRequest | None = ..., run_backend: global___BackendRequest | None = ..., + get_proto_version: builtins.int = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "get_library_element", b"get_library_element", "index_module", b"index_module", "request", b"request", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "get_library_element", b"get_library_element", "index_module", b"index_module", "request", b"request", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["request", b"request"]) -> typing_extensions.Literal["index_module", "get_library_element", "elaborate_generator", "run_refinement", "run_backend"] | None: ... + def HasField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "get_library_element", b"get_library_element", "get_proto_version", b"get_proto_version", "index_module", b"index_module", "request", b"request", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "get_library_element", b"get_library_element", "get_proto_version", b"get_proto_version", "index_module", b"index_module", "request", b"request", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["request", b"request"]) -> typing_extensions.Literal["index_module", "get_library_element", "elaborate_generator", "run_refinement", "run_backend", "get_proto_version"] | None: ... global___HdlRequest = HdlRequest @@ -447,6 +451,7 @@ class HdlResponse(google.protobuf.message.Message): ELABORATE_GENERATOR_FIELD_NUMBER: builtins.int RUN_REFINEMENT_FIELD_NUMBER: builtins.int RUN_BACKEND_FIELD_NUMBER: builtins.int + GET_PROTO_VERSION_FIELD_NUMBER: builtins.int ERROR_FIELD_NUMBER: builtins.int @property def index_module(self) -> global___IndexResponse: @@ -459,6 +464,7 @@ class HdlResponse(google.protobuf.message.Message): def run_refinement(self) -> global___RefinementResponse: ... @property def run_backend(self) -> global___BackendResponse: ... + get_proto_version: builtins.int @property def error(self) -> global___ErrorResponse: ... def __init__( @@ -469,10 +475,11 @@ class HdlResponse(google.protobuf.message.Message): elaborate_generator: global___GeneratorResponse | None = ..., run_refinement: global___RefinementResponse | None = ..., run_backend: global___BackendResponse | None = ..., + get_proto_version: builtins.int = ..., error: global___ErrorResponse | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "error", b"error", "get_library_element", b"get_library_element", "index_module", b"index_module", "response", b"response", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "error", b"error", "get_library_element", b"get_library_element", "index_module", b"index_module", "response", b"response", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["response", b"response"]) -> typing_extensions.Literal["index_module", "get_library_element", "elaborate_generator", "run_refinement", "run_backend", "error"] | None: ... + def HasField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "error", b"error", "get_library_element", b"get_library_element", "get_proto_version", b"get_proto_version", "index_module", b"index_module", "response", b"response", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["elaborate_generator", b"elaborate_generator", "error", b"error", "get_library_element", b"get_library_element", "get_proto_version", b"get_proto_version", "index_module", b"index_module", "response", b"response", "run_backend", b"run_backend", "run_refinement", b"run_refinement"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["response", b"response"]) -> typing_extensions.Literal["index_module", "get_library_element", "elaborate_generator", "run_refinement", "run_backend", "get_proto_version", "error"] | None: ... global___HdlResponse = HdlResponse diff --git a/getting-started.md b/getting-started.md index 57cbc8273..837b7088f 100644 --- a/getting-started.md +++ b/getting-started.md @@ -86,10 +86,7 @@ However, you can't do both (since they would duplicate the same results), but yo ## A top-level design: Blinky _In this example, we will create a circuit consisting of a LED and switch connected to a microcontroller._ -For now, you will be working directly in the `PoylmorphicBlocks` repository folder directly. -In the future, we will improve the flow to allow adding `PolymorphicBlocks` as a package dependency. - -Start by opening `blinky_skeleton.py`, which is pre-populated with this skeleton code: +Start by creating a `blinky.py`, and filling it with this skeleton code: ```python from edg import * @@ -105,6 +102,11 @@ if __name__ == "__main__": compile_board_inplace(BlinkyExample) ``` +> If using the IDE, `blinky.py` must be within the project directory. +> Make sure you've set up the project according to the [setup document](setup.md). +> +> If using the command line, `blinky.py` can be created anywhere. + - `from edg import *` brings in the base classes for circuit construction, like `SimpleBoardTop`. - `class BlinkyExample` contains the (top-level) circuit you're going to build, and it extends the top-level hierarchical block base class `SimpleBoardTop`. It's empty for now, but we'll fill it in the next section. @@ -131,7 +133,7 @@ Try building the example now: > - Changes to `__init__` do not re-compile instantiating classes, even if default values have been updated. 3. The design should build, and you should get a run log that looks something like: ``` - Starting compilation of blinky_skeleton.BlinkyExample + Starting compilation of blinky.BlinkyExample Using interpreter from configured SDK [...] [... lots of compilation output here ...] Completed: generate netlist: wrote [...] @@ -140,8 +142,8 @@ Try building the example now: ![run menu](docs/ide/ide_run_config.png) -
If not using the IDE - Run `python blinkly_skeleton.py` from the command line. - If all worked, this should create a folder `PolymorphicBlocks/BlinkyExample` with a netlist `BlinkyExample.net` inside. + Run `python blinky.py` from the command line. + If all worked, this should create a folder `BlinkyExample` with a netlist `BlinkyExample.net` inside.
-
Resolving common errors @@ -153,7 +155,7 @@ Try building the example now: ### Creating the microcontroller and LED For this simple example, we connect an LED to a STM32F103 microcontroller, and have everything powered by a USB type-C receptacle. -**In `blinky_skeleton.py`, `# your implementation here`, add this code** to instantiate the microcontroller and LED as follows: +**In `blinky.py`, `# your implementation here`, add this code** to instantiate the microcontroller and LED as follows: ```diff super().contents() - # your implementation here diff --git a/proto/edgrpc/hdl.proto b/proto/edgrpc/hdl.proto index c2c88da38..109ec5073 100644 --- a/proto/edgrpc/hdl.proto +++ b/proto/edgrpc/hdl.proto @@ -120,6 +120,8 @@ message HdlRequest { GeneratorRequest elaborate_generator = 3; // returns the elaborated IR RefinementRequest run_refinement = 5; BackendRequest run_backend = 4; + + uint32 get_proto_version = 90; // no data } } @@ -131,6 +133,8 @@ message HdlResponse { RefinementResponse run_refinement = 5; BackendResponse run_backend = 4; + uint32 get_proto_version = 90; + ErrorResponse error = 99; } } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..8bba3055b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = ["setuptools>=61.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "edg" +version = "0.0.0" +description = "Hardware description language for circuit boards" +readme = "README.md" +authors = [{ name = "Ducky", email = "richard.lin@berkeley.edu" }] +license = { file = "LICENSE" } +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: BSD License", + "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)", + "Typing :: Typed", +] +keywords = ["PCB", "hardware description language"] +dependencies = [ + "protobuf >= 3.20.0", + "sexpdata==0.0.3", + "Deprecated", + "typing_extensions", +] +requires-python = ">=3.9" + +[tool.setuptools] +packages = ["edgir", "edgrpc", "edg_core", "edg_hdl_server", "electronics_model", "electronics_abstract_parts", "electronics_lib", "edg"] + +[tool.setuptools.package-data] +edg_core = ["resources/edg-compiler-precompiled.jar"] +electronics_abstract_parts = ["resources/*.kicad_sch"] +electronics_lib = ["resources/Pruned_JLCPCB SMT Parts Library(20220419).csv", "resources/*.kicad_sch"] + +[project.urls] +Homepage = "https://github.com/BerkeleyHCI/PolymorphicBlocks" diff --git a/setup.md b/setup.md index e9c2f8a19..67ce71eda 100644 --- a/setup.md +++ b/setup.md @@ -1,8 +1,10 @@ # Setup -_Setting up and using the IDE is recommended, but HDL-only setup is also documented below._ + +## HDL Core Setup +_These are the instructions to set up the HDL core, both for command-line compilation as well as using the IDE (requires additional setup in the next section)._ _Runs natively on Windows, Linux, and Mac._ -1. Download [sbt](https://www.scala-sbt.org/download.html), the Scala build tool. +1. Make sure you are using Python 3.9 (or later). 2. If you do not have a Java JDK installed, download and install one. An open-source one is [Eclipse Temurin](https://adoptium.net/temurin/releases/?version=17). Java 11 (or later) is required. @@ -23,10 +25,19 @@ _Runs natively on Windows, Linux, and Mac._ Version reporting formats are not standardized, for example Oracle's Java 8 may report as `Oracle Corporation Java 1.8.0_351`.
-3. Download or clone the IDE plugin sources from https://github.com/BerkeleyHCI/edg-ide. +3. Install the Python package using the package manager: + `pip install edg` + + +## IDE Setup +_Using the IDE is recommended since it provides a design browser and block diagram visualizer, but it's optional._ +_Currently, the IDE must be built from source (it's pretty straightforward), but in the future we may make pre-compiled versions available._ + +1. Download [sbt](https://www.scala-sbt.org/download.html), the Scala build tool. +2. Download or clone the IDE plugin sources from https://github.com/BerkeleyHCI/edg-ide. - If using command line git: make sure to initialize submodules: `git submodule update --init --recursive`. - If using GitHub Desktop: it should automatically clone submodules for you. -4. In the `edg-ide` directory, run `sbt runIDE`. +3. In the `edg-ide` directory, run `sbt runIDE`. sbt will automatically fetch dependencies, compile the plugin, and start the IDE with the plugin enabled. - The first run may take a while.
Resolving common errors @@ -41,33 +52,10 @@ _Runs natively on Windows, Linux, and Mac._ this is because your Java version is pre-11. See the section above for instructions to install a more recent JDK.
-5. In the IDE, **open the `PolymorphicBlocks` folder as a project**. - - You do not need to clone `PolymorphicBlocks` separately, you can use the submodule in `edg-ide`. -6. Once the project loads, open `blinky_skeleton.py`. -7. Set up the Python interpreter once the prompt shows up, "Configure Python interpreter". - - Recommended: set up a Virtualenv environment based on a Python 3.7 (or later) interpreter. - - Install Python requirements (packages) if prompted, this banner should show up in the `blinky_skeleton.py` editor: - ![Dependencies banner](docs/ide/ide_deps.png) - - When requirements finish installing, this popup will show up on the bottom right: - ![Install complete popup](docs/ide/ide_deps_complete.png) -
Using other interpreters - - - If using Pipenv (may need to be installed separately), IntelliJ should also prompt you to install dependencies similarly to the Virtualenv case above. - - If using System Interpreter or Conda: you will need to install dependencies manually, `pip install -r requirements.txt`. - - On Ubuntu, you may need to select a particular version of Python for pip, using `python3.8 -m pip` instead of `pip` directly. -
-9. Open the Block Visualizer tab on the right. - It will be empty right now. - - -
Alternatively, HDL-only setup... - -_This isn't necessary if you're using the IDE._ -_Runs natively on Windows, Linux, and Mac._ - -1. Make sure you are using Python 3.7 (or later). -2. Make sure you have Java 11 or later. -3. Install the needed dependencies. - If using pip: `pip install -r requirements.txt` - - If on Linux and you get an error along the lines of `python: command not found`, you may need to `apt install python-is-python3`. -
+4. Within the IDE, create a new Python project. + - The location can be anywhere. + - The default environment type of Virtualenv is fine. + - **Make sure to check "inherit global site-packages"**, so that the pip-installed package will be visible. + > A prior version of this setup guide had the IDE open the PolymorphicBlocks repository and develop directly on it. + > Unless you are developing the core HDL or core libraries, this pip package + new project setup flow is recommended. + > However, the IDE does also support working with the PolymorphicBlocks project.