diff --git a/cmake/CompileActorCompiler.cmake b/cmake/CompileActorCompiler.cmake index 0b0c1e6d345..1e3c12d19f6 100644 --- a/cmake/CompileActorCompiler.cmake +++ b/cmake/CompileActorCompiler.cmake @@ -1,20 +1,35 @@ +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +find_program(MCS_EXECUTABLE mcs) +find_program(MONO_EXECUTABLE mono) + +set(ACTORCOMPILER_PY_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/__main__.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/errors.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/actor_parser.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/actor_compiler.py) + set(ACTORCOMPILER_CSPROJ ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actorcompiler.csproj) + set(ACTORCOMPILER_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorCompiler.cs ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorParser.cs ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ParseTree.cs ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Program.cs ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Properties/AssemblyInfo.cs) + set(ACTOR_COMPILER_REFERENCES "-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml" ) +add_custom_target(actorcompiler_py DEPENDS ${ACTORCOMPILER_PY_SRCS}) + if(WIN32) - add_executable(actorcompiler ${ACTORCOMPILER_SRCS}) - target_compile_options(actorcompiler PRIVATE "/langversion:6") + add_executable(actorcompiler_csharp ${ACTORCOMPILER_SRCS}) + target_compile_options(actorcompiler_csharp PRIVATE "/langversion:6") set_property( - TARGET actorcompiler + TARGET actorcompiler_csharp PROPERTY VS_DOTNET_REFERENCES "System" "System.Core" @@ -23,6 +38,10 @@ if(WIN32) "Microsoft.CSharp" "System.Data" "System.Xml") + set(ACTORCOMPILER_CSHARP_COMMAND $ + CACHE INTERNAL "Command to run the C# actor compiler") + add_custom_target(actorcompiler) + add_dependencies(actorcompiler actorcompiler_csharp actorcompiler_py) elseif(CSHARP_USE_MONO) add_custom_command( OUTPUT actorcompiler.exe @@ -31,11 +50,23 @@ elseif(CSHARP_USE_MONO) DEPENDS ${ACTORCOMPILER_SRCS} COMMENT "Compile actor compiler" VERBATIM) - add_custom_target(actorcompiler + add_custom_target(actorcompiler_csharp DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe) set(actor_exe "${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe") + set(ACTORCOMPILER_CSHARP_COMMAND ${MONO_EXECUTABLE} ${actor_exe} + CACHE INTERNAL "Command to run the C# actor compiler") + add_custom_target(actorcompiler) + add_dependencies(actorcompiler actorcompiler_csharp actorcompiler_py) else() dotnet_build(${ACTORCOMPILER_CSPROJ} SOURCE ${ACTORCOMPILER_SRCS}) set(actor_exe "${actorcompiler_EXECUTABLE_PATH}") message(STATUS "Actor compiler path: ${actor_exe}") + # dotnet_build already creates a target named 'actorcompiler', so we just add Python dependency + add_dependencies(actorcompiler actorcompiler_py) + set(ACTORCOMPILER_CSHARP_COMMAND ${actor_exe} + CACHE INTERNAL "Command to run the C# actor compiler") endif() + +set(ACTORCOMPILER_COMMAND + ${Python3_EXECUTABLE} -m flow.actorcompiler_py + CACHE INTERNAL "Command to run the actor compiler") diff --git a/cmake/FlowCommands.cmake b/cmake/FlowCommands.cmake index dadbab42fdc..cee4727871c 100644 --- a/cmake/FlowCommands.cmake +++ b/cmake/FlowCommands.cmake @@ -263,25 +263,21 @@ function(add_flow_target) endforeach() list(APPEND generated_files ${out_file}) - if(WIN32) - add_custom_command( - OUTPUT "${out_file}" - COMMAND $ "${in_file}" "${out_file}" - ${actor_compiler_flags} + if(ACTORCOMPILER_CSHARP_COMMAND) + set(py_out_file "${out_file}.py_gen") + set(cs_out_file "${out_file}.cs_gen") + add_custom_command(OUTPUT "${out_file}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CMAKE_SOURCE_DIR}" + ${ACTORCOMPILER_COMMAND} "${in_file}" "${py_out_file}" ${actor_compiler_flags} + COMMAND ${ACTORCOMPILER_CSHARP_COMMAND} "${in_file}" "${cs_out_file}" ${actor_compiler_flags} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/flow/actorcompiler_py/compare_actor_output.py "${cs_out_file}" "${py_out_file}" + COMMAND ${CMAKE_COMMAND} -E copy "${py_out_file}" "${out_file}" DEPENDS "${in_file}" actorcompiler - COMMENT "Compile actor: ${src}") - elseif(CSHARP_USE_MONO) - add_custom_command( - OUTPUT "${out_file}" - COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${in_file}" "${out_file}" - ${actor_compiler_flags} > /dev/null - DEPENDS "${in_file}" actorcompiler - COMMENT "Compile actor: ${src}") + COMMENT "Compile and compare actor: ${src}") else() - add_custom_command( - OUTPUT "${out_file}" - COMMAND ${actor_exe} "${in_file}" "${out_file}" - ${actor_compiler_flags} > /dev/null + add_custom_command(OUTPUT "${out_file}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CMAKE_SOURCE_DIR}" + ${ACTORCOMPILER_COMMAND} "${in_file}" "${out_file}" ${actor_compiler_flags} DEPENDS "${in_file}" actorcompiler COMMENT "Compile actor: ${src}") endif() diff --git a/flow/actorcompiler_py/Actor checklist.txt b/flow/actorcompiler_py/Actor checklist.txt new file mode 100644 index 00000000000..82ed087cefc --- /dev/null +++ b/flow/actorcompiler_py/Actor checklist.txt @@ -0,0 +1,84 @@ +Compile issues: + +- wait() must always assign the resulting value to a newly declared variable. + +- Variables used across a "wait() boundary" must be declared state + + +Remember to: + +- Add useful ASSERT()s + +- Add BUGGIFY() statements to expose rare cases to simulation + +- Add TEST() statements to any conditional/rare cases + +- Comment invariants, strategy, preconditions, tricky stuff when you + figure it out, even if it's not "your" code. + +- Factor common asynchronous control flows to use composition + of generic actors such as: + waitForAll + timeout + splitFuture + recurring + smartQuorum + AsyncMap + broadcast + &&, || + etc... + +- Declare classes NonCopyable unless they are, and you know what + that means + + +Run time issues: + +- Is the actor return type "void"? Make sure that some exception or timeout + will trigger eventually to clean up the actor. + +- If you send a future to another future, a long-lived forwardPromise + actor is created--make sure that the event happens eventually to free + this actor. + +- If you return a Future instead of a T from an actor, a forwardPromise + actor is created with the same lifetime issues as above. + +- Remember that parameters are internally passed to the actor as const & + and then copied into actor state variables + +- When you use *GetReply() or LoadBalance(), the "server" responding to + your request might get your request multiple times. + +- When you use getReply() instead of tryGetReply() you must ensure that the + actor will be cancelled if the service you are trying to connect to is + no longer available. (Otherwise, an infinite waiting loop) + +- For each wait: + + - An actor_cancelled exception can be thrown if the actor's return + value future is dropped. + + - An exception can arrive instead of a value + + - What happens if it never returns? + + - If the client fulfilling the wait is coming over the network, you + might get the same request multiple times + + +Performance issues: + +- Wait a little extra time before doing something time-consuming or + irreversible to see if it is still necessary. + +- When waiting for a number of things, wait a little extra time to get + the stragglers. (See the SmartQuorum() generic actor) + +- If asking another asynchronous server to do units of work, don't queue up more + work than is necessary to keep the server busy. Likewise, if you are + busy, let your own work queue fill up to signal your requester + that you are blocked. Also do this personally with managers assigning + you stuff. + +- Pass all variables as "const &" if their size is greater than 8 bytes. diff --git a/flow/actorcompiler_py/__main__.py b/flow/actorcompiler_py/__main__.py new file mode 100644 index 00000000000..b6a03591a27 --- /dev/null +++ b/flow/actorcompiler_py/__main__.py @@ -0,0 +1,101 @@ +""" +Allow running the actorcompiler as a module: + python3 -m flow.actorcompiler input.actor.cpp output.g.cpp +""" +from __future__ import annotations + +import argparse +import os +import stat +import sys +from pathlib import Path + +from .actor_parser import ActorParser, ErrorMessagePolicy +from .errors import ActorCompilerError + + +def overwrite_by_move(target: Path, temporary: Path) -> None: + if target.exists(): + target.chmod(stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + target.unlink() + os.replace(temporary, target) + target.chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="actorcompiler", + description="Python port of the Flow actor compiler", + add_help=False, + usage="actorcompiler [--disable-diagnostics] [--generate-probes]", + ) + parser.add_argument("input", nargs="?") + parser.add_argument("output", nargs="?") + parser.add_argument("--disable-diagnostics", action="store_true") + parser.add_argument("--generate-probes", action="store_true") + parser.add_argument("--help", action="help", help=argparse.SUPPRESS) + args = parser.parse_args() + if not args.input or not args.output: + parser.print_usage(sys.stderr) + sys.exit(100) + return args + + +def main() -> int: + args = parse_arguments() + input_path = Path(args.input) + output_path = Path(args.output) + output_tmp = output_path.with_suffix(output_path.suffix + ".tmp") + output_uid = output_path.with_suffix(output_path.suffix + ".uid") + + policy = ErrorMessagePolicy() + policy.disable_diagnostics = args.disable_diagnostics + + try: + print("actorcompiler", " ".join(sys.argv[1:])) + text = input_path.read_text() + parser = ActorParser( + text, + str(input_path).replace("\\", "/"), + policy, + args.generate_probes, + ) + + with output_tmp.open("w", newline="\n") as out_file: + parser.write(out_file, str(output_path).replace("\\", "/")) + overwrite_by_move(output_path, output_tmp) + + with output_tmp.open("w", newline="\n") as uid_file: + for (hi, lo), value in parser.uid_objects.items(): + uid_file.write(f"{hi}|{lo}|{value}\n") + overwrite_by_move(output_uid, output_tmp) + + return 0 + except ActorCompilerError as exc: + print( + f"{input_path}({exc.source_line}): error FAC1000: {exc}", + file=sys.stderr, + ) + if output_tmp.exists(): + output_tmp.unlink() + if output_path.exists(): + output_path.chmod(stat.S_IWUSR | stat.S_IRUSR) + output_path.unlink() + return 1 + except Exception as exc: # pylint: disable=broad-except + import traceback + traceback.print_exc() + print( + f"{input_path}(1): error FAC2000: Internal {exc}", + file=sys.stderr, + ) + if output_tmp.exists(): + output_tmp.unlink() + if output_path.exists(): + output_path.chmod(stat.S_IWUSR | stat.S_IRUSR) + output_path.unlink() + return 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/flow/actorcompiler_py/actor_compiler.py b/flow/actorcompiler_py/actor_compiler.py new file mode 100644 index 00000000000..5a6a6c6e9a4 --- /dev/null +++ b/flow/actorcompiler_py/actor_compiler.py @@ -0,0 +1,1583 @@ +from __future__ import annotations + +import hashlib +import io +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + +from .errors import ActorCompilerError +from .parse_tree import ( + Actor, + BreakStatement, + ChooseStatement, + CodeBlock, + ContinueStatement, + Declaration, + Descr, + ForStatement, + IfStatement, + LoopStatement, + PlainOldCodeStatement, + RangeForStatement, + ReturnStatement, + Statement, + StateDeclarationStatement, + ThrowStatement, + TryStatement, + VarDeclaration, + WaitStatement, + WhenStatement, + WhileStatement, +) + + +class Function: + def __init__( + self, + name: str = "", + return_type: str = "int", + formal_parameters: Optional[Sequence[str]] = None, + ) -> None: + self.name = name + self.return_type = return_type + self.formal_parameters = list(formal_parameters or []) + self.end_is_unreachable = False + self.exception_parameter_is: Optional[str] = None + self.public_name = False + self.specifiers = "" + self._indentation = "" + self._body = io.StringIO() + self.was_called = False + self.overload: Optional["Function"] = None + + def set_overload(self, overload: "Function") -> None: + self.overload = overload + + def pop_overload(self) -> Optional["Function"]: + overload = self.overload + self.overload = None + return overload + + def add_overload(self, *formal_parameters: str) -> None: + overload = Function( + name=self.name, + return_type=self.return_type, + formal_parameters=formal_parameters, + ) + overload.end_is_unreachable = self.end_is_unreachable + overload._indentation = self._indentation + self.set_overload(overload) + + def indent(self, change: int) -> None: + if change > 0: + self._indentation += "\t" * change + elif change < 0: + self._indentation = self._indentation[:change] + if self.overload is not None: + self.overload.indent(change) + + def write_line_unindented(self, text: str) -> None: + self._body.write(f"{text}\n") + if self.overload is not None: + self.overload.write_line_unindented(text) + + def write_line(self, text: str, *args: object) -> None: + if args: + text = text.format(*args) + self._body.write(f"{self._indentation}{text}\n") + if self.overload is not None: + self.overload.write_line(text) + + @property + def body_text(self) -> str: + value = self._body.getvalue() + if self.overload is not None: + _ = self.overload.body_text + return value + + def use_by_name(self) -> str: + self.was_called = True + return self.name if self.public_name else f"a_{self.name}" + + def call(self, *parameters: str) -> str: + params = ", ".join(parameters) + return f"{self.use_by_name()}({params})" + + +class LiteralBreak(Function): + def __init__(self) -> None: + super().__init__(name="break!", return_type="") + + def call(self, *parameters: str) -> str: + if parameters: + raise ActorCompilerError(0, "LiteralBreak called with parameters!") + self.was_called = True + return "break" + + +class LiteralContinue(Function): + def __init__(self) -> None: + super().__init__(name="continue!", return_type="") + + def call(self, *parameters: str) -> str: + if parameters: + raise ActorCompilerError(0, "LiteralContinue called with parameters!") + self.was_called = True + return "continue" + + +@dataclass +class StateVar: + type: str = "" + name: str = "" + initializer: Optional[str] = None + initializer_constructor_syntax: bool = False + source_line: int = 0 + + +@dataclass +class CallbackVar: + type: str = "" + callback_group: int = 0 + source_line: int = 0 + + +class Context: + def __init__( + self, + target: Optional[Function] = None, + next_func: Optional[Function] = None, + break_f: Optional[Function] = None, + continue_f: Optional[Function] = None, + catch_f_err: Optional[Function] = None, + try_loop_depth: int = 0, + ) -> None: + self.target = target + self.next = next_func + self.break_f = break_f + self.continue_f = continue_f + self.catch_f_err = catch_f_err + self.try_loop_depth = try_loop_depth + + def unreachable(self) -> None: + self.target = None + + def with_target(self, new_target: Function) -> "Context": + return Context( + target=new_target, + break_f=self.break_f, + continue_f=self.continue_f, + catch_f_err=self.catch_f_err, + try_loop_depth=self.try_loop_depth, + ) + + def loop_context( + self, + new_target: Function, + break_f: Function, + continue_f: Function, + deltaLoopDepth: int, + ) -> "Context": + return Context( + target=new_target, + break_f=break_f, + continue_f=continue_f, + catch_f_err=self.catch_f_err, + try_loop_depth=self.try_loop_depth + deltaLoopDepth, + ) + + def with_catch(self, new_catch: Function) -> "Context": + return Context( + target=self.target, + break_f=self.break_f, + continue_f=self.continue_f, + catch_f_err=new_catch, + ) + + def clone(self) -> "Context": + return Context( + target=self.target, + next_func=self.next, + break_f=self.break_f, + continue_f=self.continue_f, + catch_f_err=self.catch_f_err, + try_loop_depth=self.try_loop_depth, + ) + + def copy_from(self, other: "Context") -> None: + self.target = other.target + self.next = other.next + self.break_f = other.break_f + self.continue_f = other.continue_f + self.catch_f_err = other.catch_f_err + self.try_loop_depth = other.try_loop_depth + + +class DescrCompiler: + def __init__(self, descr: "Descr", brace_depth: int) -> None: + self.descr = descr + self.member_indent_str = "\t" * brace_depth + + def write(self, writer) -> int: + lines = 0 + indent = self.member_indent_str + writer.write( + f"{indent}template<> struct Descriptor {{\n" + ) + writer.write( + f'{indent}\tstatic StringRef typeName() {{ return "{self.descr.name}"_sr; }}\n' + ) + writer.write(f"{indent}\ttypedef {self.descr.name} type;\n") + lines += 3 + for dec in self.descr.body: + writer.write(f"{indent}\tstruct {dec.name}Descriptor {{\n") + writer.write( + f'{indent}\t\tstatic StringRef name() {{ return "{dec.name}"_sr; }}\n' + ) + writer.write( + f'{indent}\t\tstatic StringRef typeName() {{ return "{dec.type}"_sr; }}\n' + ) + writer.write( + f'{indent}\t\tstatic StringRef comment() {{ return "{dec.comment}"_sr; }}\n' + ) + writer.write(f"{indent}\t\ttypedef {dec.type} type;\n") + writer.write( + f"{indent}\t\tstatic inline type get({self.descr.name}& from);\n" + ) + writer.write(f"{indent}\t}};\n") + lines += 7 + writer.write(f"{indent}\ttypedef std::tuple<") + first = True + for dec in self.descr.body: + if not first: + writer.write(",") + writer.write(f"{dec.name}Descriptor") + first = False + writer.write("> fields;\n") + writer.write( + f"{indent}\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type field_indexes;\n" + ) + writer.write(f"{indent}}};\n") + if self.descr.super_class_list: + writer.write( + f"{indent}struct {self.descr.name} : {self.descr.super_class_list} {{\n" + ) + else: + writer.write(f"{indent}struct {self.descr.name} {{\n") + lines += 4 + for dec in self.descr.body: + writer.write(f"{indent}\t{dec.type} {dec.name}; //{dec.comment}\n") + lines += 1 + writer.write(f"{indent}}};\n") + lines += 1 + for dec in self.descr.body: + writer.write( + f"{indent}{dec.type} Descriptor<{self.descr.name}>::{dec.name}Descriptor::get({self.descr.name}& from) {{ return from.{dec.name}; }}\n" + ) + lines += 1 + return lines + + +class ActorCompiler: + member_indent_str = "\t" + loop_depth_0 = "int loopDepth=0" + loop_depth = "int loopDepth" + code_indent = 2 + used_class_names: set[str] = set() + + def __init__( + self, + actor: Actor, + source_file: str, + is_top_level: bool, + line_numbers_enabled: bool, + generate_probes: bool, + ) -> None: + self.actor = actor + self.source_file = source_file + self.is_top_level = is_top_level + self.line_numbers_enabled = line_numbers_enabled + self.generate_probes = generate_probes + self.class_name = "" + self.full_class_name = "" + self.state_class_name = "" + self.state: List[StateVar] = [] + self.callbacks: List[CallbackVar] = [] + self.choose_groups = 0 + self.when_count = 0 + self.this = "" + self.uid_objects: Dict[Tuple[int, int], str] = {} + self.functions: Dict[str, Function] = {} + self.iterators: Dict[str, int] = {} + self.find_state() + + def byte_to_long(self, data: bytes) -> int: + result = 0 + for b in data: + result += b + result <<= 8 + return result & ((1 << 64) - 1) + + def get_uid_from_string(self, value: str) -> Tuple[int, int]: + digest = hashlib.sha256(value.encode("utf-8")).digest() + first = self.byte_to_long(digest[:8]) + second = self.byte_to_long(digest[8:16]) + return (first, second) + + def writeActorFunction(self, writer, full_return_type: str) -> None: + self.writeTemplate(writer) + self.line_number(writer, self.actor.source_line) + for attribute in self.actor.attributes: + writer.write(f"{attribute} ") + if self.actor.is_static: + writer.write("static ") + namespace_prefix = ( + "" if self.actor.name_space is None else f"{self.actor.name_space}::" + ) + params = ", ".join(self.parameter_list()) + writer.write( + f"{full_return_type} {namespace_prefix}{self.actor.name}( {params} ) {{\n" + ) + self.line_number(writer, self.actor.source_line) + ctor_args = ", ".join(param.name for param in self.actor.parameters) + new_actor = f"new {self.full_class_name}({ctor_args})" + if self.actor.return_type is not None: + writer.write(f"\treturn Future<{self.actor.return_type}>({new_actor});\n") + else: + writer.write(f"\t{new_actor};\n") + writer.write("}\n") + + def writeActorClass( + self, writer, full_state_class_name: str, body: Function + ) -> None: + writer.write( + f"// This generated class is to be used only via {self.actor.name}()\n" + ) + self.writeTemplate(writer) + self.line_number(writer, self.actor.source_line) + callback_bases = ", ".join(f"public {cb.type}" for cb in self.callbacks) + if callback_bases: + callback_bases += ", " + writer.write( + "class {0} final : public Actor<{2}>, {3}public FastAllocated<{1}>, public {4} {{\n".format( + self.class_name, + self.full_class_name, + "void" if self.actor.return_type is None else self.actor.return_type, + callback_bases, + full_state_class_name, + ) + ) + writer.write("public:\n") + writer.write(f"\tusing FastAllocated<{self.full_class_name}>::operator new;\n") + writer.write(f"\tusing FastAllocated<{self.full_class_name}>::operator delete;\n") + actor_identifier_key = f"{self.source_file}:{self.actor.name}" + uid = self.get_uid_from_string(actor_identifier_key) + self.uid_objects[(uid[0], uid[1])] = actor_identifier_key + writer.write( + f"\tstatic constexpr ActorIdentifier __actorIdentifier = UID({uid[0]}UL, {uid[1]}UL);\n" + ) + writer.write("\tActiveActorHelper activeActorHelper;\n") + writer.write("#pragma clang diagnostic push\n") + writer.write('#pragma clang diagnostic ignored "-Wdelete-non-virtual-dtor"\n') + if self.actor.return_type is not None: + writer.write(" void destroy() override {\n") + writer.write(" activeActorHelper.~ActiveActorHelper();\n") + writer.write( + f" static_cast*>(this)->~Actor();\n" + ) + writer.write(" operator delete(this);\n") + writer.write(" }\n") + else: + writer.write(" void destroy() {{\n") + writer.write(" activeActorHelper.~ActiveActorHelper();\n") + writer.write(" static_cast*>(this)->~Actor();\n") + writer.write(" operator delete(this);\n") + writer.write(" }}\n") + writer.write("#pragma clang diagnostic pop\n") + for cb in self.callbacks: + writer.write(f"friend struct {cb.type};\n") + self.line_number(writer, self.actor.source_line) + self.writeConstructor(body, writer, full_state_class_name) + self.writeCancelFunc(writer) + writer.write("};\n") + + def write(self, writer) -> None: + full_return_type = ( + f"Future<{self.actor.return_type}>" + if self.actor.return_type is not None + else "void" + ) + for i in range(1 << 16): + class_name = "{3}{0}{1}Actor{2}".format( + self.actor.name[:1].upper(), + self.actor.name[1:], + str(i) if i != 0 else "", + ( + self.actor.enclosing_class.replace("::", "_") + "_" + if self.actor.enclosing_class is not None + and self.actor.is_forward_declaration + else ( + self.actor.name_space.replace("::", "_") + "_" + if self.actor.name_space is not None + else "" + ) + ), + ) + if self.actor.is_forward_declaration: + self.class_name = class_name + break + if class_name not in ActorCompiler.used_class_names: + self.class_name = class_name + ActorCompiler.used_class_names.add(class_name) + break + self.full_class_name = self.class_name + self.get_template_actuals() + actor_class_formal = VarDeclaration(name=self.class_name, type="class") + self.this = f"static_cast<{actor_class_formal.name}*>(this)" + self.state_class_name = self.class_name + "State" + full_state = self.state_class_name + self.get_template_actuals( + VarDeclaration(type="class", name=self.full_class_name) + ) + if self.actor.is_forward_declaration: + for attribute in self.actor.attributes: + writer.write(f"{attribute} ") + if self.actor.is_static: + writer.write("static ") + ns = "" if self.actor.name_space is None else f"{self.actor.name_space}::" + writer.write( + f"{full_return_type} {ns}{self.actor.name}( {', '.join(self.parameter_list())} );\n" + ) + if self.actor.enclosing_class is not None: + writer.write(f"template friend class {self.state_class_name};\n") + return + body = self.get_function("", "body", self.loop_depth_0) + body_context = Context( + target=body, + catch_f_err=self.get_function( + body.name, "Catch", "Error error", self.loop_depth_0 + ), + ) + end_context = self.try_catch_compile(self.actor.body, body_context) + if end_context.target is not None: + if self.actor.return_type is None: + self.compile_statement( + ReturnStatement( + first_source_line=self.actor.source_line, expression="" + ), + end_context, + ) + else: + raise ActorCompilerError( + self.actor.source_line, + "Actor {0} fails to return a value", + self.actor.name, + ) + if self.actor.return_type is not None: + body_context.catch_f_err.write_line(f"this->~{self.state_class_name}();") + body_context.catch_f_err.write_line( + f"{self.this}->sendErrorAndDelPromiseRef(error);" + ) + else: + body_context.catch_f_err.write_line(f"delete {self.this};") + body_context.catch_f_err.write_line("loopDepth = 0;") + if self.is_top_level and self.actor.name_space is None: + writer.write("namespace {\n") + writer.write( + f"// This generated class is to be used only via {self.actor.name}()\n" + ) + self.writeTemplate(writer, actor_class_formal) + self.line_number(writer, self.actor.source_line) + writer.write(f"class {self.state_class_name} {{\n") + writer.write("public:\n") + self.line_number(writer, self.actor.source_line) + self.writeStateConstructor(writer) + self.writeStateDestructor(writer) + self.writeFunctions(writer) + for st in self.state: + self.line_number(writer, st.source_line) + writer.write(f"\t{st.type} {st.name};\n") + writer.write("};\n") + self.writeActorClass(writer, full_state, body) + if self.is_top_level and self.actor.name_space is None: + writer.write("} // namespace\n") + self.writeActorFunction(writer, full_return_type) + if self.actor.test_case_parameters is not None: + writer.write( + f"ACTOR_TEST_CASE({self.actor.name}, {self.actor.test_case_parameters})\n" + ) + + thisAddress = "reinterpret_cast(this)" + + def probe_enter(self, fun: Function, name: str, index: int = -1) -> None: + if self.generate_probes: + fun.write_line( + 'fdb_probe_actor_enter("{0}", {1}, {2});', + name, + self.thisAddress, + index, + ) + block_identifier = self.get_uid_from_string(fun.name) + fun.write_line("#ifdef WITH_ACAC") + fun.write_line( + "static constexpr ActorBlockIdentifier __identifier = UID({0}UL, {1}UL);", + block_identifier[0], + block_identifier[1], + ) + fun.write_line( + "ActorExecutionContextHelper __helper(static_cast<{0}*>(this)->activeActorHelper.actorID, __identifier);", + self.class_name, + ) + fun.write_line("#endif // WITH_ACAC") + + def probe_exit(self, fun: Function, name: str, index: int = -1) -> None: + if self.generate_probes: + fun.write_line( + 'fdb_probe_actor_exit("{0}", {1}, {2});', + name, + self.thisAddress, + index, + ) + + def probe_create(self, fun: Function, name: str) -> None: + if self.generate_probes: + fun.write_line( + 'fdb_probe_actor_create("{0}", {1});', name, self.thisAddress + ) + + def probe_destroy(self, fun: Function, name: str) -> None: + if self.generate_probes: + fun.write_line( + 'fdb_probe_actor_destroy("{0}", {1});', + name, + self.thisAddress, + ) + + def line_number(self, writer, source_line: int) -> None: + if source_line == 0: + raise ActorCompilerError(0, "Invalid source line (0)") + if self.line_numbers_enabled: + writer.write( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( + source_line, self.source_file + ) + ) + + def line_numberFunction(self, func: Function, source_line: int) -> None: + if source_line == 0: + raise ActorCompilerError(0, "Invalid source line (0)") + if self.line_numbers_enabled: + func.write_line_unindented( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"'.format( + source_line, self.source_file + ) + ) + + def try_catch( + self, + cx: Context, + catch_function: Optional[Function], + catch_loop_depth: int, + action, + use_loop_depth: bool = True, + ) -> None: + if catch_function is not None: + cx.target.write_line("try {") + cx.target.indent(+1) + action() + if catch_function is not None: + cx.target.indent(-1) + cx.target.write_line("}") + cx.target.write_line("catch (Error& error) {") + if use_loop_depth: + cx.target.write_line( + "\tloopDepth = {0};", + catch_function.call( + "error", self.adjust_loop_depth(catch_loop_depth) + ), + ) + else: + cx.target.write_line("\t{0};", catch_function.call("error", "0")) + cx.target.write_line("} catch (...) {") + if use_loop_depth: + cx.target.write_line( + "\tloopDepth = {0};", + catch_function.call( + "unknown_error()", self.adjust_loop_depth(catch_loop_depth) + ), + ) + else: + cx.target.write_line( + "\t{0};", catch_function.call("unknown_error()", "0") + ) + cx.target.write_line("}") + + def try_catch_compile(self, block: CodeBlock, cx: Context) -> Context: + result_holder = {"ctx": cx} + + def action() -> None: + result_holder["ctx"] = self._try_catch_compile_body(block, cx) + + self.try_catch(cx, cx.catch_f_err, cx.try_loop_depth, action) + return result_holder["ctx"] + + def _try_catch_compile_body(self, block: CodeBlock, cx: Context) -> Context: + compiled = self.compile(block, cx, True) + if compiled.target is not None: + next_func = self.get_function(compiled.target.name, "cont", self.loop_depth) + compiled.target.write_line("loopDepth = {0};", next_func.call("loopDepth")) + compiled.target = next_func + compiled.next = None + return compiled + + def writeTemplate(self, writer, *extra_parameters: VarDeclaration) -> None: + formals = list(self.actor.template_formals or []) + list(extra_parameters) + if not formals: + return + self.line_number(writer, self.actor.source_line) + writer.write( + "template <{0}>\n".format(", ".join(f"{p.type} {p.name}" for p in formals)) + ) + + def get_template_actuals(self, *extra_parameters: VarDeclaration) -> str: + formals = list(self.actor.template_formals or []) + list(extra_parameters) + if not formals: + return "" + return "<{0}>".format(", ".join(p.name for p in formals)) + + def will_continue(self, stmt: Statement) -> bool: + return any( + isinstance(sub, (ChooseStatement, WaitStatement, TryStatement)) + for sub in self.flatten(stmt) + ) + + def as_code_block(self, statement: Statement) -> CodeBlock: + if isinstance(statement, CodeBlock): + return statement + return CodeBlock(statements=[statement]) + + def flatten(self, stmt: Optional[Statement]) -> Iterable[Statement]: + if stmt is None: + return [] + + def _flatten(s: Statement) -> Iterable[Statement]: + yield s + if isinstance(s, LoopStatement): + yield from self.flatten(s.body) + elif isinstance(s, WhileStatement): + yield from self.flatten(s.body) + elif isinstance(s, ForStatement): + yield from self.flatten(s.body) + elif isinstance(s, RangeForStatement): + yield from self.flatten(s.body) + elif isinstance(s, CodeBlock): + for child in s.statements: + yield from self.flatten(child) + elif isinstance(s, IfStatement): + yield from self.flatten(s.if_body) + if s.else_body: + yield from self.flatten(s.else_body) + elif isinstance(s, ChooseStatement): + yield from self.flatten(s.body) + elif isinstance(s, WhenStatement): + if s.body: + yield from self.flatten(s.body) + elif isinstance(s, TryStatement): + yield from self.flatten(s.try_body) + for c in s.catches: + yield from self.flatten(c.body) + + return list(_flatten(stmt)) + + def find_state(self) -> None: + self.state = [ + StateVar( + source_line=self.actor.source_line, + name=p.name, + type=p.type, + initializer=p.name, + ) + for p in self.actor.parameters + ] + + def adjust_loop_depth(self, subtract: int) -> str: + if subtract == 0: + return "loopDepth" + return f"std::max(0, loopDepth - {subtract})" + + def parameter_list(self) -> List[str]: + params = [] + for p in self.actor.parameters: + if p.initializer: + params.append(f"{p.type} const& {p.name} = {p.initializer}") + else: + params.append(f"{p.type} const& {p.name}") + return params + + def get_function( + self, + base_name: str, + add_name: str, + *formal_parameters: str, + overload_formal_parameters: Optional[Sequence[str]] = None, + ) -> Function: + if len(formal_parameters) == 1 and isinstance( + formal_parameters[0], (list, tuple) + ): + params = list(formal_parameters[0]) + else: + params = [p for p in formal_parameters if p] + + if add_name == "cont" and len(base_name) >= 5 and base_name[-5:-1] == "cont": + proposed_name = base_name[:-1] + else: + proposed_name = base_name + add_name + idx = 1 + while f"{proposed_name}{idx}" in self.functions: + idx += 1 + func = Function( + name=f"{proposed_name}{idx}", + return_type="int", + formal_parameters=params, + ) + func.indent(self.code_indent) + if overload_formal_parameters: + func.add_overload(*overload_formal_parameters) + self.functions[func.name] = func + return func + + def writeFunctions(self, writer) -> None: + for func in self.functions.values(): + body = func.body_text + if body: + self.writeFunction(writer, func, body) + if func.overload: + overload_body = func.overload.body_text + if overload_body: + self.writeFunction(writer, func.overload, overload_body) + + def writeFunction(self, writer, func: Function, body: str) -> None: + spec = "" if not func.specifiers else f" {func.specifiers}" + trailing = " " if spec == "" else "" + signature = ( + f"{self.member_indent_str}" + f"{'' if func.return_type == '' else f'{func.return_type} '}" + f"{func.use_by_name()}({','.join(func.formal_parameters)})" + f"{spec}{trailing}\n" + ) + writer.write(signature) + if func.return_type != "": + writer.write(f"{self.member_indent_str}{{\n") + writer.write(body) + writer.write("\n") + if not func.end_is_unreachable: + writer.write(f"{self.member_indent_str}\treturn loopDepth;\n") + writer.write(f"{self.member_indent_str}}}\n") + + def writeCancelFunc(self, writer) -> None: + if not self.actor.is_cancellable(): + return + cancel_func = Function( + name="cancel", + return_type="void", + formal_parameters=[], + ) + cancel_func.end_is_unreachable = True + cancel_func.public_name = True + cancel_func.specifiers = "override" + cancel_func.indent(self.code_indent) + cancel_func.write_line("auto wait_state = this->actor_wait_state;") + cancel_func.write_line("this->actor_wait_state = -1;") + cancel_func.write_line("switch (wait_state) {") + last_group = -1 + for cb in sorted(self.callbacks, key=lambda c: c.callback_group): + if cb.callback_group != last_group: + last_group = cb.callback_group + cancel_func.write_line( + "case {0}: this->a_callback_error(({1}*)0, actor_cancelled()); break;", + cb.callback_group, + cb.type, + ) + cancel_func.write_line("}") + self.writeFunction(writer, cancel_func, cancel_func.body_text) + + def writeConstructor( + self, body: Function, writer, full_state_class_name: str + ) -> None: + constructor = Function( + name=self.class_name, + return_type="", + formal_parameters=self.parameter_list(), + ) + constructor.end_is_unreachable = True + constructor.public_name = True + constructor.indent(self.code_indent) + constructor.write_line( + " : Actor<{0}>(),".format( + "void" if self.actor.return_type is None else self.actor.return_type + ) + ) + constructor.write_line( + " {0}({1}),".format( + full_state_class_name, + ", ".join(p.name for p in self.actor.parameters), + ) + ) + constructor.write_line(" activeActorHelper(__actorIdentifier)") + constructor.indent(-1) + constructor.write_line("{") + constructor.indent(+1) + self.probe_enter(constructor, self.actor.name) + constructor.write_line("#ifdef ENABLE_SAMPLING") + constructor.write_line('this->lineage.setActorName("{0}");', self.actor.name) + constructor.write_line("LineageScope _(&this->lineage);") + constructor.write_line("#endif") + constructor.write_line("this->{0};", body.call()) + self.probe_exit(constructor, self.actor.name) + self.writeFunction(writer, constructor, constructor.body_text) + + def writeStateConstructor(self, writer) -> None: + constructor = Function( + name=self.state_class_name, + return_type="", + formal_parameters=self.parameter_list(), + ) + constructor.end_is_unreachable = True + constructor.public_name = True + constructor.indent(self.code_indent) + ini = None + line = self.actor.source_line + for state_var in self.state: + if state_var.initializer is None: + continue + self.line_numberFunction(constructor, line) + if ini is None: + ini = " : " + else: + constructor.write_line(ini + ",") + ini = " " + ini += f"{state_var.name}({state_var.initializer})" + line = state_var.source_line + self.line_numberFunction(constructor, line) + if ini: + constructor.write_line(ini) + constructor.indent(-1) + constructor.write_line("{") + constructor.indent(+1) + self.probe_create(constructor, self.actor.name) + self.writeFunction(writer, constructor, constructor.body_text) + + def writeStateDestructor(self, writer) -> None: + destructor = Function( + name=f"~{self.state_class_name}", + return_type="", + formal_parameters=[], + ) + destructor.end_is_unreachable = True + destructor.public_name = True + destructor.indent(self.code_indent) + destructor.indent(-1) + destructor.write_line("{") + destructor.indent(+1) + self.probe_destroy(destructor, self.actor.name) + self.writeFunction(writer, destructor, destructor.body_text) + + def compile( + self, block: CodeBlock, context: Context, ok_to_continue: bool = True + ) -> Context: + cx = context.clone() + cx.next = None + for stmt in block.statements: + if cx.target is None: + raise ActorCompilerError(stmt.first_source_line, "Unreachable code.") + if cx.next is None: + cx.next = self.get_function(cx.target.name, "cont", self.loop_depth) + self.compile_statement(stmt, cx) + if cx.next.was_called: + if cx.target is None: + raise ActorCompilerError( + stmt.first_source_line, "Unreachable continuation called?" + ) + if not ok_to_continue: + raise ActorCompilerError( + stmt.first_source_line, "Unexpected continuation" + ) + cx.target = cx.next + cx.next = None + return cx + + def compile_statement(self, stmt: Statement, cx: Context) -> None: + handler_name = f"_compile_{stmt.__class__.__name__}" + handler = getattr(self, handler_name, None) + if handler is None: + raise ActorCompilerError( + stmt.first_source_line, + "Statement type {0} not supported yet.", + stmt.__class__.__name__, + ) + handler(stmt, cx) + + def _is_top_level_state_decl(self, stmt: StateDeclarationStatement) -> bool: + for candidate in self.actor.body.statements: + if not isinstance(candidate, StateDeclarationStatement): + break + if candidate is stmt: + return True + return False + + def _compile_PlainOldCodeStatement( + self, stmt: PlainOldCodeStatement, cx: Context + ) -> None: + self.line_numberFunction(cx.target, stmt.first_source_line) + cx.target.write_line(stmt.code) + + def _compile_StateDeclarationStatement( + self, stmt: StateDeclarationStatement, cx: Context + ) -> None: + if self._is_top_level_state_decl(stmt): + self.state.append( + StateVar( + source_line=stmt.first_source_line, + name=stmt.decl.name, + type=stmt.decl.type, + initializer=stmt.decl.initializer, + initializer_constructor_syntax=stmt.decl.initializer_constructor_syntax, + ) + ) + else: + self.state.append( + StateVar( + source_line=stmt.first_source_line, + name=stmt.decl.name, + type=stmt.decl.type, + initializer=None, + ) + ) + if stmt.decl.initializer is not None: + self.line_numberFunction(cx.target, stmt.first_source_line) + if ( + stmt.decl.initializer_constructor_syntax + or stmt.decl.initializer == "" + ): + cx.target.write_line( + "{0} = {1}({2});", + stmt.decl.name, + stmt.decl.type, + stmt.decl.initializer, + ) + else: + cx.target.write_line( + "{0} = {1};", stmt.decl.name, stmt.decl.initializer + ) + + def get_iterator_name(self, cx: Context) -> str: + name = f"RangeFor{cx.target.name}Iterator" + self.iterators.setdefault(name, 0) + idx = self.iterators[name] + self.iterators[name] += 1 + return f"{name}{idx}" + + def emit_native_loop( + self, source_line: int, head: str, body: Statement, cx: Context + ) -> bool: + self.line_numberFunction(cx.target, source_line) + cx.target.write_line(head + " {") + cx.target.indent(+1) + literal_break = LiteralBreak() + literal_continue = LiteralContinue() + self.compile( + self.as_code_block(body), + cx.loop_context(cx.target, literal_break, literal_continue, 0), + True, + ) + cx.target.indent(-1) + cx.target.write_line("}") + return not literal_break.was_called + + def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: + no_condition = stmt.cond_expression in ("", "true", "1") + if not self.will_continue(stmt.body): + if ( + self.emit_native_loop( + stmt.first_source_line, + f"for({stmt.init_expression};{stmt.cond_expression};{stmt.next_expression})", + stmt.body, + cx, + ) + and no_condition + ): + cx.unreachable() + return + + init_stmt = PlainOldCodeStatement( + code=f"{stmt.init_expression};", first_source_line=stmt.first_source_line + ) + self._compile_PlainOldCodeStatement(init_stmt, cx) + + if no_condition: + full_body = stmt.body + else: + condition = IfStatement( + expression=f"!({stmt.cond_expression})", + if_body=BreakStatement(first_source_line=stmt.first_source_line), + first_source_line=stmt.first_source_line, + ) + full_body = CodeBlock( + statements=[condition] + list(self.as_code_block(stmt.body).statements), + first_source_line=stmt.first_source_line, + ) + + loopF = self.get_function(cx.target.name, "loopHead", self.loop_depth) + loopBody = self.get_function(cx.target.name, "loopBody", self.loop_depth) + break_f = self.get_function(cx.target.name, "break", self.loop_depth) + continue_f = ( + loopF + if stmt.next_expression == "" + else self.get_function(cx.target.name, "continue", self.loop_depth) + ) + + loopF.write_line("int oldLoopDepth = ++loopDepth;") + loopF.write_line( + "while (loopDepth == oldLoopDepth) loopDepth = {0};", + loopBody.call("loopDepth"), + ) + + endLoop = self.compile( + self.as_code_block(full_body), + cx.loop_context(loopBody, break_f, continue_f, +1), + True, + ).target + + if endLoop is not None and endLoop is not loopBody: + if stmt.next_expression: + self._compile_PlainOldCodeStatement( + PlainOldCodeStatement( + code=f"{stmt.next_expression};", + first_source_line=stmt.first_source_line, + ), + cx.with_target(endLoop), + ) + endLoop.write_line("if (loopDepth == 0) return {0};", loopF.call("0")) + + cx.target.write_line("loopDepth = {0};", loopF.call("loopDepth")) + + if continue_f is not loopF and continue_f.was_called: + self._compile_PlainOldCodeStatement( + PlainOldCodeStatement( + code=f"{stmt.next_expression};", first_source_line=stmt.first_source_line + ), + cx.with_target(continue_f), + ) + continue_f.write_line("if (loopDepth == 0) return {0};", loopF.call("0")) + + if break_f.was_called: + self.try_catch( + cx.with_target(break_f), + cx.catch_f_err, + cx.try_loop_depth, + lambda: break_f.write_line("return {0};", cx.next.call("loopDepth")), + ) + else: + cx.unreachable() + + def _compile_RangeForStatement(self, stmt: RangeForStatement, cx: Context) -> None: + if self.will_continue(stmt.body): + container = next( + (s for s in self.state if s.name == stmt.range_expression), None + ) + if container is None: + raise ActorCompilerError( + stmt.first_source_line, + "container of range-based for with continuation must be a state variable", + ) + iterator_name = self.get_iterator_name(cx) + self.state.append( + StateVar( + source_line=stmt.first_source_line, + name=iterator_name, + type=f"decltype(std::begin(std::declval<{container.type}>()))", + initializer=None, + ) + ) + equivalent = ForStatement( + init_expression=f"{iterator_name} = std::begin({stmt.range_expression})", + cond_expression=f"{iterator_name} != std::end({stmt.range_expression})", + next_expression=f"++{iterator_name}", + first_source_line=stmt.first_source_line, + body=CodeBlock( + statements=[ + PlainOldCodeStatement( + code=f"{stmt.range_decl} = *{iterator_name};", + first_source_line=stmt.first_source_line, + ), + stmt.body, + ], + first_source_line=stmt.first_source_line, + ), + ) + self._compile_ForStatement(equivalent, cx) + else: + self.emit_native_loop( + stmt.first_source_line, + f"for( {stmt.range_decl} : {stmt.range_expression} )", + stmt.body, + cx, + ) + + def _compile_WhileStatement(self, stmt: WhileStatement, cx: Context) -> None: + equivalent = ForStatement( + init_expression="", + cond_expression=stmt.expression, + next_expression="", + body=stmt.body, + first_source_line=stmt.first_source_line, + ) + self._compile_ForStatement(equivalent, cx) + + def _compile_LoopStatement(self, stmt: LoopStatement, cx: Context) -> None: + equivalent = ForStatement( + init_expression="", + cond_expression="", + next_expression="", + body=stmt.body, + first_source_line=stmt.first_source_line, + ) + self._compile_ForStatement(equivalent, cx) + + def _compile_BreakStatement(self, stmt: BreakStatement, cx: Context) -> None: + if cx.break_f is None: + raise ActorCompilerError(stmt.first_source_line, "break outside loop") + if isinstance(cx.break_f, LiteralBreak): + cx.target.write_line("{0};", cx.break_f.call()) + else: + cx.target.write_line( + "return {0}; // break", cx.break_f.call("loopDepth==0?0:loopDepth-1") + ) + cx.unreachable() + + def _compile_ContinueStatement(self, stmt: ContinueStatement, cx: Context) -> None: + if cx.continue_f is None: + raise ActorCompilerError(stmt.first_source_line, "continue outside loop") + if isinstance(cx.continue_f, LiteralContinue): + cx.target.write_line("{0};", cx.continue_f.call()) + else: + cx.target.write_line( + "return {0}; // continue", cx.continue_f.call("loopDepth") + ) + cx.unreachable() + + def _compile_CodeBlock(self, stmt: CodeBlock, cx: Context) -> None: + cx.target.write_line("{") + cx.target.indent(+1) + end = self.compile(stmt, cx, True) + cx.target.indent(-1) + cx.target.write_line("}") + if end.target is None: + cx.unreachable() + elif end.target is not cx.target: + end.target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + + def _compile_ReturnStatement(self, stmt: ReturnStatement, cx: Context) -> None: + self.line_numberFunction(cx.target, stmt.first_source_line) + if (stmt.expression == "") != (self.actor.return_type is None): + raise ActorCompilerError( + stmt.first_source_line, + "Return statement does not match actor declaration", + ) + if self.actor.return_type is not None: + if stmt.expression == "Never()": + cx.target.write_line("this->~{0}();", self.state_class_name) + cx.target.write_line("{0}->sendAndDelPromiseRef(Never());", self.this) + else: + cx.target.write_line( + "if (!{0}->SAV<{1}>::futures) {{ (void)({2}); this->~{3}(); {0}->destroy(); return 0; }}", + self.this, + self.actor.return_type, + stmt.expression, + self.state_class_name, + ) + if any(s.name == stmt.expression for s in self.state): + cx.target.write_line( + "new (&{0}->SAV< {1} >::value()) {1}(std::move({2})); // state_var_RVO", + self.this, + self.actor.return_type, + stmt.expression, + ) + else: + cx.target.write_line( + "new (&{0}->SAV< {1} >::value()) {1}({2});", + self.this, + self.actor.return_type, + stmt.expression, + ) + cx.target.write_line("this->~{0}();", self.state_class_name) + cx.target.write_line("{0}->finishSendAndDelPromiseRef();", self.this) + else: + cx.target.write_line("delete {0};", self.this) + cx.target.write_line("return 0;") + cx.unreachable() + + def _compile_ThrowStatement(self, stmt: ThrowStatement, cx: Context) -> None: + self.line_numberFunction(cx.target, stmt.first_source_line) + if stmt.expression == "": + if cx.target.exception_parameter_is is not None: + cx.target.write_line( + "return {0};", + cx.catch_f_err.call( + cx.target.exception_parameter_is, + self.adjust_loop_depth(cx.try_loop_depth), + ), + ) + else: + raise ActorCompilerError( + stmt.first_source_line, + "throw statement with no expression has no current exception in scope", + ) + else: + cx.target.write_line( + "return {0};", + cx.catch_f_err.call( + stmt.expression, self.adjust_loop_depth(cx.try_loop_depth) + ), + ) + cx.unreachable() + + def _compile_IfStatement(self, stmt: IfStatement, cx: Context) -> None: + use_continuation = self.will_continue(stmt.if_body) or ( + stmt.else_body is not None and self.will_continue(stmt.else_body) + ) + self.line_numberFunction(cx.target, stmt.first_source_line) + constexpr = "constexpr " if stmt.constexpr else "" + cx.target.write_line(f"if {constexpr}({stmt.expression})") + cx.target.write_line("{") + cx.target.indent(+1) + if_target = self.compile( + self.as_code_block(stmt.if_body), cx, use_continuation + ).target + if use_continuation and if_target is not None: + if_target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + cx.target.indent(-1) + cx.target.write_line("}") + else_target = None + if stmt.else_body is not None or use_continuation: + cx.target.write_line("else") + cx.target.write_line("{") + cx.target.indent(+1) + else_target = cx.target + if stmt.else_body is not None: + else_target = self.compile( + self.as_code_block(stmt.else_body), cx, use_continuation + ).target + if use_continuation and else_target is not None: + else_target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + cx.target.indent(-1) + cx.target.write_line("}") + if if_target is None and stmt.else_body is not None and else_target is None: + cx.unreachable() + elif not cx.next.was_called and use_continuation: + raise ActorCompilerError( + stmt.first_source_line, "Internal error: IfStatement: next not called?" + ) + + def _compile_TryStatement(self, stmt: TryStatement, cx: Context) -> None: + if len(stmt.catches) != 1: + raise ActorCompilerError( + stmt.first_source_line, + "try statement must have exactly one catch clause", + ) + reachable = False + catch_clause = stmt.catches[0] + catch_expression = catch_clause.expression.replace(" ", "") + catch_param = "" + if catch_expression != "...": + if not catch_expression.startswith("Error&"): + raise ActorCompilerError( + catch_clause.first_source_line, + "Only type 'Error' or '...' may be caught in an actor function", + ) + catch_param = catch_expression[6:] + if not catch_param: + catch_param = "__current_error" + catch_f_err = self.get_function( + cx.target.name, "Catch", f"const Error& {catch_param}", self.loop_depth_0 + ) + catch_f_err.exception_parameter_is = catch_param + end = self.try_catch_compile( + self.as_code_block(stmt.try_body), cx.with_catch(catch_f_err) + ) + if end.target is not None: + reachable = True + self.try_catch( + end, + cx.catch_f_err, + cx.try_loop_depth, + lambda: end.target.write_line( + "loopDepth = {0};", cx.next.call("loopDepth") + ), + ) + + def handle_catch() -> None: + nonlocal reachable + cend = self._compile_try_catch_body(catch_clause, catch_f_err, cx) + if cend.target is not None: + reachable = True + + self.try_catch( + cx.with_target(catch_f_err), + cx.catch_f_err, + cx.try_loop_depth, + handle_catch, + ) + if not reachable: + cx.unreachable() + + def _compile_try_catch_body( + self, catch_clause: TryStatement.Catch, catch_f_err: Function, cx: Context + ) -> Context: + cend = self.compile( + self.as_code_block(catch_clause.body), cx.with_target(catch_f_err), True + ) + if cend.target is not None: + cend.target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + return cend + + def _compile_WaitStatement(self, stmt: WaitStatement, cx: Context) -> None: + equivalent = ChooseStatement( + body=CodeBlock( + statements=[ + WhenStatement( + wait=stmt, + body=None, + first_source_line=stmt.first_source_line, + ) + ], + first_source_line=stmt.first_source_line, + ), + first_source_line=stmt.first_source_line, + ) + if not stmt.result_is_state: + cx.next.formal_parameters = [ + f"{stmt.result.type} const& {stmt.result.name}", + self.loop_depth, + ] + cx.next.add_overload( + f"{stmt.result.type} && {stmt.result.name}", self.loop_depth + ) + self._compile_ChooseStatement(equivalent, cx) + + def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None: + group = self.choose_groups + 1 + self.choose_groups = group + codeblock = stmt.body if isinstance(stmt.body, CodeBlock) else None + if codeblock is None: + raise ActorCompilerError( + stmt.first_source_line, + "'choose' must be followed by a compound statement.", + ) + choices = [] + for idx, choice_stmt in enumerate(codeblock.statements): + if not isinstance(choice_stmt, WhenStatement): + raise ActorCompilerError( + choice_stmt.first_source_line, + "only 'when' statements are valid in a 'choose' block.", + ) + index = self.when_count + idx + param_prefix = "__" if choice_stmt.wait.result_is_state else "" + const_param = f"{choice_stmt.wait.result.type} const& {param_prefix}{choice_stmt.wait.result.name}" + rvalue_param = f"{choice_stmt.wait.result.type} && {param_prefix}{choice_stmt.wait.result.name}" + body_func = self.get_function( + cx.target.name, + "when", + [const_param, self.loop_depth], + overload_formal_parameters=[rvalue_param, self.loop_depth], + ) + future_name = f"__when_expr_{index}" + callback_template = ( + "ActorSingleCallback" + if choice_stmt.wait.is_wait_next + else "ActorCallback" + ) + callback_type = f"{callback_template}< {self.full_class_name}, {index}, {choice_stmt.wait.result.type} >" + callback_type_state = f"{callback_template}< {self.class_name}, {index}, {choice_stmt.wait.result.type} >" + choices.append( + { + "Stmt": choice_stmt, + "Group": group, + "Index": index, + "Body": body_func, + "Future": future_name, + "CallbackType": callback_type, + "CallbackTypeState": callback_type_state, + } + ) + self.when_count += len(choices) + exit_func = self.get_function("exitChoose", "", []) + exit_func.return_type = "void" + exit_func.write_line( + "if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", self.this + ) + for choice in choices: + exit_func.write_line( + "{0}->{1}::remove();", self.this, choice["CallbackTypeState"] + ) + exit_func.end_is_unreachable = True + + reachable = False + for choice in choices: + self.callbacks.append( + CallbackVar( + source_line=choice["Stmt"].first_source_line, + callback_group=choice["Group"], + type=choice["CallbackType"], + ) + ) + r = choice["Body"] + if choice["Stmt"].wait.result_is_state: + overload = r.pop_overload() + decl_stmt = StateDeclarationStatement( + decl=VarDeclaration( + type=choice["Stmt"].wait.result.type, + name=choice["Stmt"].wait.result.name, + initializer=f"__{choice['Stmt'].wait.result.name}", + initializer_constructor_syntax=False, + ), + first_source_line=choice["Stmt"].first_source_line, + ) + self._compile_StateDeclarationStatement(decl_stmt, cx.with_target(r)) + if overload is not None: + overload.write_line( + "{0} = std::move(__{0});", + choice["Stmt"].wait.result.name, + ) + r.set_overload(overload) + if choice["Stmt"].body is not None: + r = self.compile( + self.as_code_block(choice["Stmt"].body), cx.with_target(r), True + ).target + if r is not None: + reachable = True + if len(cx.next.formal_parameters) == 1: + r.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + else: + overload = r.pop_overload() + r.write_line( + "loopDepth = {0};", + cx.next.call(choice["Stmt"].wait.result.name, "loopDepth"), + ) + if overload is not None: + overload.write_line( + "loopDepth = {0};", + cx.next.call( + f"std::move({choice['Stmt'].wait.result.name})", + "loopDepth", + ), + ) + r.set_overload(overload) + + cb_func = Function( + name="callback_fire", + return_type="void", + formal_parameters=[ + f"{choice['CallbackTypeState']}*", + f"{choice['Stmt'].wait.result.type} const& value", + ], + ) + cb_func.end_is_unreachable = True + cb_func.add_overload( + f"{choice['CallbackTypeState']}*", + f"{choice['Stmt'].wait.result.type} && value", + ) + cb_func.indent(self.code_indent) + self.probe_enter(cb_func, self.actor.name, choice["Index"]) + cb_func.write_line("{0};", exit_func.call()) + overload_func = cb_func.pop_overload() + + def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None: + self.try_catch( + cx.with_target(target_func), + cx.catch_f_err, + cx.try_loop_depth, + lambda ch=choice_dict, tf=target_func, expression=expr: tf.write_line( + "{0};", ch["Body"].call(expression, "0") + ), + False, + ) + + fire_body(cb_func, "value") + if overload_func is not None: + fire_body(overload_func, "std::move(value)") + cb_func.set_overload(overload_func) + self.probe_exit(cb_func, self.actor.name, choice["Index"]) + self.functions[f"{cb_func.name}#{choice['Index']}"] = cb_func + + err_func = Function( + name="callback_error", + return_type="void", + formal_parameters=[f"{choice['CallbackTypeState']}*", "Error err"], + ) + err_func.end_is_unreachable = True + err_func.indent(self.code_indent) + self.probe_enter(err_func, self.actor.name, choice["Index"]) + err_func.write_line("{0};", exit_func.call()) + self.try_catch( + cx.with_target(err_func), + cx.catch_f_err, + cx.try_loop_depth, + lambda: err_func.write_line("{0};", cx.catch_f_err.call("err", "0")), + False, + ) + self.probe_exit(err_func, self.actor.name, choice["Index"]) + self.functions[f"{err_func.name}#{choice['Index']}"] = err_func + + first_choice = True + for choice in choices: + get_func = "pop" if choice["Stmt"].wait.is_wait_next else "get" + self.line_numberFunction(cx.target, choice["Stmt"].wait.first_source_line) + if choice["Stmt"].wait.is_wait_next: + cx.target.write_line( + "auto {0} = {1};", + choice["Future"], + choice["Stmt"].wait.future_expression, + ) + cx.target.write_line( + 'static_assert(std::is_same>::value || std::is_same>::value, "invalid type");', + choice["Future"], + choice["Stmt"].wait.result.type, + ) + else: + cx.target.write_line( + "StrictFuture<{0}> {1} = {2};", + choice["Stmt"].wait.result.type, + choice["Future"], + choice["Stmt"].wait.future_expression, + ) + if first_choice: + first_choice = False + self.line_numberFunction(cx.target, stmt.first_source_line) + if self.actor.is_cancellable(): + cx.target.write_line( + "if ({1}->actor_wait_state < 0) return {0};", + cx.catch_f_err.call( + "actor_cancelled()", self.adjust_loop_depth(cx.try_loop_depth) + ), + self.this, + ) + cx.target.write_line( + "if ({0}.isReady()) {{ if ({0}.isError()) return {2}; else return {1}; }};", + choice["Future"], + choice["Body"].call(f"{choice['Future']}.{get_func}()", "loopDepth"), + cx.catch_f_err.call( + f"{choice['Future']}.getError()", + self.adjust_loop_depth(cx.try_loop_depth), + ), + ) + cx.target.write_line("{1}->actor_wait_state = {0};", group, self.this) + for choice in choices: + self.line_numberFunction(cx.target, choice["Stmt"].wait.first_source_line) + cx.target.write_line( + "{0}.addCallbackAndClear(static_cast<{1}*>({2}));", + choice["Future"], + choice["CallbackTypeState"], + self.this, + ) + cx.target.write_line("loopDepth = 0;") + if not reachable: + cx.unreachable() diff --git a/flow/actorcompiler_py/actor_parser.py b/flow/actorcompiler_py/actor_parser.py new file mode 100644 index 00000000000..f6941f85d43 --- /dev/null +++ b/flow/actorcompiler_py/actor_parser.py @@ -0,0 +1,1126 @@ +from __future__ import annotations + +import io +import re +import sys +from dataclasses import dataclass +from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple + +from .errors import ActorCompilerError +from .actor_compiler import ActorCompiler, DescrCompiler + +from .parse_tree import ( + Actor, + BreakStatement, + ChooseStatement, + CodeBlock, + ContinueStatement, + Declaration, + Descr, + ForStatement, + IfStatement, + LoopStatement, + PlainOldCodeStatement, + RangeForStatement, + ReturnStatement, + StateDeclarationStatement, + Statement, + ThrowStatement, + TryStatement, + VarDeclaration, + WaitStatement, + WhenStatement, + WhileStatement, +) + + +class ErrorMessagePolicy: + def __init__(self) -> None: + self.disable_diagnostics = False + + def handle_actor_without_wait(self, source_file: str, actor: Actor) -> None: + if not self.disable_diagnostics and not actor.is_test_case: + print( + f"{source_file}:{actor.source_line}: warning: ACTOR {actor.name} does not contain a wait() statement", + file=sys.stderr, + ) + + def actors_no_discard_by_default(self) -> bool: + return not self.disable_diagnostics + + +@dataclass +class Token: + value: str + position: int = 0 + source_line: int = 0 + brace_depth: int = 0 + paren_depth: int = 0 + + @property + def is_whitespace(self) -> bool: + return ( + self.value in (" ", "\n", "\r", "\r\n", "\t") + or self.value.startswith("//") + or self.value.startswith("/*") + ) + + def ensure(self, error: str, pred) -> "Token": + if not pred(self): + raise ActorCompilerError(self.source_line, error) + return self + + def get_matching_range_in(self, token_range: "TokenRange") -> "TokenRange": + if self.value == "<": + sub = TokenRange(token_range.tokens, self.position, token_range.end_pos) + gen = AngleBracketParser.not_inside_angle_brackets(sub) + next(gen, None) # skip the "<" + closing = next(gen, None) + if closing is None: + raise ActorCompilerError(self.source_line, "Syntax error: Unmatched <") + return TokenRange(token_range.tokens, self.position + 1, closing.position) + if self.value == "[": + sub = TokenRange(token_range.tokens, self.position, token_range.end_pos) + gen = BracketParser.not_inside_brackets(sub) + next(gen, None) # skip the "[" + closing = next(gen, None) + if closing is None: + raise ActorCompilerError(self.source_line, "Syntax error: Unmatched [") + return TokenRange(token_range.tokens, self.position + 1, closing.position) + + pairs = {"(": ")", ")": "(", "{": "}", "}": "{"} + if self.value not in pairs: + raise RuntimeError("Can't match this token") + pred = ( + (lambda t: t.value != ")" or t.paren_depth != self.paren_depth) + if self.value == "(" + else ( + (lambda t: t.value != "(" or t.paren_depth != self.paren_depth) + if self.value == ")" + else ( + (lambda t: t.value != "}" or t.brace_depth != self.brace_depth) + if self.value == "{" + else (lambda t: t.value != "{" or t.brace_depth != self.brace_depth) + ) + ) + ) + direction = 1 if self.value in ("(", "{") else -1 + if direction == -1: + rng = token_range.sub_range(token_range.begin_pos, self.position).Revtake_while(pred) + if rng.begin_pos == token_range.begin_pos: + raise ActorCompilerError( + self.source_line, f"Syntax error: Unmatched {self.value}" + ) + return rng + rng = token_range.sub_range(self.position + 1, token_range.end_pos).take_while(pred) + if rng.end_pos == token_range.end_pos: + raise ActorCompilerError( + self.source_line, f"Syntax error: Unmatched {self.value}" + ) + return rng + + +class TokenRange: + def __init__(self, tokens: List[Token], begin: int, end: int) -> None: + if begin > end: + raise RuntimeError("Invalid TokenRange") + self.tokens = tokens + self.begin_pos = begin + self.end_pos = end + + def __iter__(self) -> Iterator[Token]: + for i in range(self.begin_pos, self.end_pos): + yield self.tokens[i] + + def first(self, predicate=None) -> Token: + if self.begin_pos == self.end_pos: + raise RuntimeError("Empty TokenRange") + if predicate is None: + return self.tokens[self.begin_pos] + for t in self: + if predicate(t): + return t + raise RuntimeError("Matching token not found") + + def last(self, predicate=None) -> Token: + if self.begin_pos == self.end_pos: + raise RuntimeError("Empty TokenRange") + if predicate is None: + return self.tokens[self.end_pos - 1] + for i in range(self.end_pos - 1, self.begin_pos - 1, -1): + if predicate(self.tokens[i]): + return self.tokens[i] + raise RuntimeError("Matching token not found") + + def skip(self, count: int) -> "TokenRange": + return TokenRange(self.tokens, self.begin_pos + count, self.end_pos) + + def consume(self, value_or_error, predicate=None) -> "TokenRange": + if predicate is None: + self.first().ensure( + f"Expected {value_or_error}", lambda t: t.value == value_or_error + ) + else: + self.first().ensure(value_or_error, predicate) + return self.skip(1) + + def skipWhile(self, predicate) -> "TokenRange": + e = self.begin_pos + while e < self.end_pos and predicate(self.tokens[e]): + e += 1 + return TokenRange(self.tokens, e, self.end_pos) + + def take_while(self, predicate) -> "TokenRange": + e = self.begin_pos + while e < self.end_pos and predicate(self.tokens[e]): + e += 1 + return TokenRange(self.tokens, self.begin_pos, e) + + def Revtake_while(self, predicate) -> "TokenRange": + e = self.end_pos - 1 + while e >= self.begin_pos and predicate(self.tokens[e]): + e -= 1 + return TokenRange(self.tokens, e + 1, self.end_pos) + + def Revskip_while(self, predicate) -> "TokenRange": + e = self.end_pos - 1 + while e >= self.begin_pos and predicate(self.tokens[e]): + e -= 1 + return TokenRange(self.tokens, self.begin_pos, e + 1) + + def sub_range(self, begin: int, end: int) -> "TokenRange": + return TokenRange(self.tokens, begin, end) + + def is_empty(self) -> bool: + return self.begin_pos == self.end_pos + + def length(self) -> int: + return self.end_pos - self.begin_pos + + def all_match(self, predicate) -> bool: + return all(predicate(t) for t in self) + + def any_match(self, predicate) -> bool: + return any(predicate(t) for t in self) + + +class BracketParser: + @staticmethod + def not_inside_brackets(tokens: Iterable[Token]) -> Iterator[Token]: + bracket_depth = 0 + base_pd = None + for tok in tokens: + if base_pd is None: + base_pd = tok.paren_depth + if tok.paren_depth == base_pd and tok.value == "]": + bracket_depth -= 1 + if bracket_depth == 0: + yield tok + if tok.paren_depth == base_pd and tok.value == "[": + bracket_depth += 1 + + +class AngleBracketParser: + @staticmethod + def not_inside_angle_brackets(tokens: Iterable[Token]) -> Iterator[Token]: + angle_depth = 0 + base_pd = None + for tok in tokens: + if base_pd is None: + base_pd = tok.paren_depth + if tok.paren_depth == base_pd and tok.value == ">": + angle_depth -= 1 + if angle_depth == 0: + yield tok + if tok.paren_depth == base_pd and tok.value == "<": + angle_depth += 1 + + +class ActorParser: + token_expressions = [ + r"\{", + r"\}", + r"\(", + r"\)", + r"\[", + r"\]", + r"//[^\n]*", + r"/[*]([*][^/]|[^*])*[*]/", + r"'(\\.|[^\'\n])*'", + r'"(\\.|[^"\n])*"', + r"[a-zA-Z_][a-zA-Z_0-9]*", + r"\r\n", + r"\n", + r"::", + r":", + r"#[a-z]*", + r".", + ] + + identifier_pattern = re.compile(r"^[a-zA-Z_][a-zA-Z_0-9]*$") + + def __init__( + self, + text: str, + source_file: str, + error_message_policy: ErrorMessagePolicy, + generate_probes: bool, + ) -> None: + self.source_file = source_file + self.error_message_policy = error_message_policy + self.generate_probes = generate_probes + self.tokens = [Token(value=t) for t in self.tokenize(text)] + self.line_numbers_enabled = True + self.uid_objects: Dict[Tuple[int, int], str] = {} + self.token_array = self.tokens + self.count_parens() + + def tokenize(self, text: str) -> List[str]: + regexes = [re.compile(pattern, re.S) for pattern in self.token_expressions] + pos = 0 + tokens: List[str] = [] + while pos < len(text): + for regex in regexes: + m = regex.match(text, pos) + if m: + tokens.append(m.group(0)) + pos += len(m.group(0)) + break + else: + raise RuntimeError(f"Can't tokenize! {pos}") + return tokens + + def count_parens(self) -> None: + brace_depth = 0 + paren_depth = 0 + line_count = 1 + last_paren = None + last_brace = None + for i, token in enumerate(self.tokens): + value = token.value + if value == "}": + brace_depth -= 1 + elif value == ")": + paren_depth -= 1 + elif value == "\r\n": + line_count += 1 + elif value == "\n": + line_count += 1 + + if brace_depth < 0: + raise ActorCompilerError(line_count, "Mismatched braces") + if paren_depth < 0: + raise ActorCompilerError(line_count, "Mismatched parenthesis") + + token.position = i + token.source_line = line_count + token.brace_depth = brace_depth + token.paren_depth = paren_depth + + if value.startswith("/*"): + line_count += value.count("\n") + if value == "{": + brace_depth += 1 + if brace_depth == 1: + last_brace = token + elif value == "(": + paren_depth += 1 + if paren_depth == 1: + last_paren = token + if brace_depth != 0: + raise ActorCompilerError( + last_brace.source_line if last_brace else line_count, "Unmatched brace" + ) + if paren_depth != 0: + raise ActorCompilerError( + last_paren.source_line if last_paren else line_count, + "Unmatched parenthesis", + ) + + def write(self, writer: io.TextIOBase, destFileName: str) -> None: + ActorCompiler.used_class_names.clear() + writer.write("#define POST_ACTOR_COMPILER 1\n") + outLine = 1 + if self.line_numbers_enabled: + writer.write(f'#line {self.tokens[0].source_line} "{self.source_file}"\n') + outLine += 1 + inBlocks = 0 + classContextStack: List[Tuple[str, int]] = [] + i = 0 + while i < len(self.tokens): + tok = self.tokens[i] + if tok.source_line == 0: + raise RuntimeError("Invalid source line (0)") + if tok.value in ("ACTOR", "SWIFT_ACTOR", "TEST_CASE"): + actor = self.parse_actor(i) + end = self._parse_end + if classContextStack: + actor.enclosing_class = "::".join( + name for name, _ in classContextStack + ) + actor_writer = io.StringIO() + actorCompiler = ActorCompiler( + actor, + self.source_file, + inBlocks == 0, + self.line_numbers_enabled, + self.generate_probes, + ) + actorCompiler.write(actor_writer) + for key, value in actorCompiler.uid_objects.items(): + self.uid_objects.setdefault(key, value) + actor_lines = actor_writer.getvalue().split("\n") + hasLineNumber = False + hadLineNumber = True + for line in actor_lines: + if self.line_numbers_enabled: + isLine = "#line" in line + if isLine: + hadLineNumber = True + if not isLine and not hasLineNumber and hadLineNumber: + writer.write( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( + outLine + 1, destFileName + ) + ) + outLine += 1 + hadLineNumber = False + hasLineNumber = isLine + writer.write(line.rstrip("\n\r") + "\n") + outLine += 1 + i = end + if i < len(self.tokens) and self.line_numbers_enabled: + writer.write( + f'#line {self.tokens[i].source_line} "{self.source_file}"\n' + ) + outLine += 1 + elif tok.value == "DESCR": + descr, end = self.parse_descr(i) + descr_writer = io.StringIO() + lines = DescrCompiler(descr, tok.brace_depth).write(descr_writer) + writer.write(descr_writer.getvalue()) + outLine += lines + i = end + if i < len(self.tokens) and self.line_numbers_enabled: + writer.write( + f'#line {self.tokens[i].source_line} "{self.source_file}"\n' + ) + outLine += 1 + elif tok.value in ("class", "struct", "union"): + writer.write(tok.value) + success, name = self.parse_class_context( + self.range(i + 1, len(self.tokens)) + ) + if success: + classContextStack.append((name, inBlocks)) + else: + if tok.value == "{": + inBlocks += 1 + elif tok.value == "}": + inBlocks -= 1 + if classContextStack and classContextStack[-1][1] == inBlocks: + classContextStack.pop() + writer.write(tok.value) + outLine += tok.value.count("\n") + i += 1 + + # Parsing helpers + + def range(self, begin: int, end: int) -> TokenRange: + return TokenRange(self.tokens, begin, end) + + def parse_actor(self, pos: int) -> Actor: + token = self.tokens[pos] + actor = Actor() + actor.source_line = token.source_line + toks = self.range(pos + 1, len(self.tokens)) + heading = toks.take_while(lambda t: t.value != "{") + toSemicolon = toks.take_while(lambda t: t.value != ";") + actor.is_forward_declaration = toSemicolon.length() < heading.length() + if actor.is_forward_declaration: + heading = toSemicolon + if token.value in ("ACTOR", "SWIFT_ACTOR"): + self.parse_actorHeading(actor, heading) + else: + token.ensure("ACTOR expected!", lambda _: False) + self._parse_end = heading.end_pos + 1 + else: + body = self.range(heading.end_pos + 1, len(self.tokens)).take_while( + lambda t: t.brace_depth > toks.first().brace_depth + ) + if token.value in ("ACTOR", "SWIFT_ACTOR"): + self.parse_actorHeading(actor, heading) + elif token.value == "TEST_CASE": + self.parse_test_caseHeading(actor, heading) + actor.is_test_case = True + else: + token.ensure("ACTOR or TEST_CASE expected!", lambda _: False) + actor.body = self.parse_code_block(body) + if not actor.body.contains_wait(): + self.error_message_policy.handle_actor_without_wait(self.source_file, actor) + self._parse_end = body.end_pos + 1 + return actor + + def parse_descr(self, pos: int) -> Tuple[Descr, int]: + descr = Descr() + toks = self.range(pos + 1, len(self.tokens)) + heading = toks.take_while(lambda t: t.value != "{") + body = self.range(heading.end_pos + 1, len(self.tokens)).take_while( + lambda t: t.brace_depth > toks.first().brace_depth or t.value == ";" + ) + self.parse_descrHeading(descr, heading) + descr.body = self.parse_descrCodeBlock(body) + end = body.end_pos + 1 + return descr, end + + def parse_descrHeading(self, descr: Descr, toks: TokenRange) -> None: + nonWhitespace = lambda t: not t.is_whitespace + toks.first(nonWhitespace).ensure( + "non-struct DESCR!", lambda t: t.value == "struct" + ) + toks = ( + toks.skipWhile(lambda t: t.is_whitespace) + .skip(1) + .skipWhile(lambda t: t.is_whitespace) + ) + colon = next((t for t in toks if t.value == ":"), None) + if colon: + descr.super_class_list = self.str( + self.range(colon.position + 1, toks.end_pos) + ).strip() + toks = self.range(toks.begin_pos, colon.position) + descr.name = self.str(toks).strip() + + def parse_descrCodeBlock(self, toks: TokenRange) -> List[Declaration]: + declarations: List[Declaration] = [] + while True: + delim = next((t for t in toks if t.value == ";"), None) + if delim is None: + break + pos = delim.position + 1 + potential_comment = self.range(pos, toks.end_pos).skipWhile( + lambda t: t.value in ("\t", " ") + ) + if ( + not potential_comment.is_empty() + and potential_comment.first().value.startswith("//") + ): + pos = potential_comment.first().position + 1 + self.parse_declarationRange(self.range(toks.begin_pos, pos), declarations) + toks = self.range(pos, toks.end_pos) + if not toks.all_match(lambda t: t.is_whitespace): + raise ActorCompilerError( + toks.first(lambda t: not t.is_whitespace).source_line, + "Trailing unterminated statement in code block", + ) + return declarations + + def parse_declarationRange( + self, toks: TokenRange, declarations: List[Declaration] + ) -> None: + delim = toks.first(lambda t: t.value == ";") + nameRange = ( + self.range(toks.begin_pos, delim.position) + .Revskip_while(lambda t: t.is_whitespace) + .Revtake_while(lambda t: not t.is_whitespace) + ) + typeRange = self.range(toks.begin_pos, nameRange.begin_pos) + commentRange = self.range(delim.position + 1, toks.end_pos) + declarations.append( + Declaration( + name=self.str(nameRange).strip(), + type=self.str(typeRange).strip(), + comment=self.str(commentRange).strip().lstrip("/"), + ) + ) + + def parse_test_caseHeading(self, actor: Actor, toks: TokenRange) -> None: + actor.is_static = True + nonWhitespace = lambda t: not t.is_whitespace + paramRange = ( + toks.last(nonWhitespace) + .ensure( + "Unexpected tokens after test case parameter list.", + lambda t: t.value == ")" and t.paren_depth == toks.first().paren_depth, + ) + .get_matching_range_in(toks) + ) + actor.test_case_parameters = self.str(paramRange) + actor.name = f"flowTestCase{toks.first().source_line}" + actor.parameters = [ + VarDeclaration( + name="params", + type="UnitTestParameters", + initializer="", + initializer_constructor_syntax=False, + ) + ] + actor.return_type = "Void" + + def parse_actorHeading(self, actor: Actor, toks: TokenRange) -> None: + nonWhitespace = lambda t: not t.is_whitespace + template = toks.first(nonWhitespace) + if template.value == "template": + templateParams = ( + self.range(template.position + 1, toks.end_pos) + .first(nonWhitespace) + .ensure("Invalid template declaration", lambda t: t.value == "<") + .get_matching_range_in(toks) + ) + actor.template_formals = [ + self.parse_var_declaration(p) + for p in self.split_parameter_list(templateParams, ",") + ] + toks = self.range(templateParams.end_pos + 1, toks.end_pos) + attribute = toks.first(nonWhitespace) + while attribute.value == "[": + contents = attribute.get_matching_range_in(toks) + as_array = list(contents) + if ( + len(as_array) < 2 + or as_array[0].value != "[" + or as_array[-1].value != "]" + ): + raise ActorCompilerError( + actor.source_line, "Invalid attribute: Expected [[...]]" + ) + actor.attributes.append( + "[" + self.str(self.normalize_whitespace(contents)) + "]" + ) + toks = self.range(contents.end_pos + 1, toks.end_pos) + attribute = toks.first(nonWhitespace) + static_keyword = toks.first(nonWhitespace) + if static_keyword.value == "static": + actor.is_static = True + toks = self.range(static_keyword.position + 1, toks.end_pos) + uncancellable = toks.first(nonWhitespace) + if uncancellable.value == "UNCANCELLABLE": + actor.set_uncancellable() + toks = self.range(uncancellable.position + 1, toks.end_pos) + paramRange = ( + toks.last(nonWhitespace) + .ensure( + "Unexpected tokens after actor parameter list.", + lambda t: t.value == ")" and t.paren_depth == toks.first().paren_depth, + ) + .get_matching_range_in(toks) + ) + actor.parameters = [ + self.parse_var_declaration(p) + for p in self.split_parameter_list(paramRange, ",") + ] + nameToken = self.range(toks.begin_pos, paramRange.begin_pos - 1).last(nonWhitespace) + actor.name = nameToken.value + return_range = self.range( + toks.first().position + 1, nameToken.position + ).skipWhile(lambda t: t.is_whitespace) + retToken = return_range.first() + if retToken.value == "Future": + ofType = ( + return_range.skip(1) + .first(nonWhitespace) + .ensure("Expected <", lambda tok: tok.value == "<") + .get_matching_range_in(return_range) + ) + actor.return_type = self.str(self.normalize_whitespace(ofType)) + toks = self.range(ofType.end_pos + 1, return_range.end_pos) + elif retToken.value == "void": + actor.return_type = None + toks = return_range.skip(1) + else: + raise ActorCompilerError( + actor.source_line, "Actor apparently does not return Future" + ) + toks = toks.skipWhile(lambda t: t.is_whitespace) + if not toks.is_empty(): + if toks.last().value == "::": + actor.name_space = self.str(self.range(toks.begin_pos, toks.end_pos - 1)) + else: + raise ActorCompilerError( + actor.source_line, + "Unrecognized tokens preceding parameter list in actor declaration", + ) + if ( + self.error_message_policy.actors_no_discard_by_default() + and "[[flow_allow_discard]]" not in actor.attributes + ): + if actor.is_cancellable(): + actor.attributes.append("[[nodiscard]]") + known_flow_attributes = {"[[flow_allow_discard]]"} + for flow_attribute in [a for a in actor.attributes if a.startswith("[[flow_")]: + if flow_attribute not in known_flow_attributes: + raise ActorCompilerError( + actor.source_line, f"Unknown flow attribute {flow_attribute}" + ) + actor.attributes = [a for a in actor.attributes if not a.startswith("[[flow_")] + + def parse_var_declaration(self, tokens: TokenRange) -> VarDeclaration: + name, typeRange, initializer, constructorSyntax = self.parse_declaration(tokens) + return VarDeclaration( + name=name.value, + type=self.str(self.normalize_whitespace(typeRange)), + initializer=( + "" + if initializer is None + else self.str(self.normalize_whitespace(initializer)) + ), + initializer_constructor_syntax=constructorSyntax, + ) + + def parse_declaration(self, tokens: TokenRange): + nonWhitespace = lambda t: not t.is_whitespace + initializer = None + beforeInitializer = tokens + constructorSyntax = False + equals = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(tokens) + if t.value == "=" and t.paren_depth == tokens.first().paren_depth + ), + None, + ) + if equals: + beforeInitializer = self.range(tokens.begin_pos, equals.position) + initializer = self.range(equals.position + 1, tokens.end_pos) + else: + paren = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(tokens) + if t.value == "(" + ), + None, + ) + if paren: + constructorSyntax = True + beforeInitializer = self.range(tokens.begin_pos, paren.position) + initializer = self.range(paren.position + 1, tokens.end_pos).take_while( + lambda t, p=paren.paren_depth: t.paren_depth > p + ) + else: + brace = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(tokens) + if t.value == "{" + ), + None, + ) + if brace: + raise ActorCompilerError( + brace.source_line, + "Uniform initialization syntax is not currently supported for state variables (use '(' instead of '}' ?)", + ) + name = beforeInitializer.last(nonWhitespace) + if beforeInitializer.begin_pos == name.position: + raise ActorCompilerError( + beforeInitializer.first().source_line, "Declaration has no type." + ) + typeRange = self.range(beforeInitializer.begin_pos, name.position) + return name, typeRange, initializer, constructorSyntax + + def normalize_whitespace(self, tokens: Iterable[Token]) -> Iterable[Token]: + inWhitespace = False + leading = True + for tok in tokens: + if not tok.is_whitespace: + if inWhitespace and not leading: + yield Token(value=" ") + inWhitespace = False + yield tok + leading = False + else: + inWhitespace = True + + def split_parameter_list( + self, toks: TokenRange, delimiter: str + ) -> Iterable[TokenRange]: + if toks.begin_pos == toks.end_pos: + return [] + ranges: List[TokenRange] = [] + while True: + comma = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(toks) + if t.value == delimiter and t.paren_depth == toks.first().paren_depth + ), + None, + ) + if comma is None: + break + ranges.append(self.range(toks.begin_pos, comma.position)) + toks = self.range(comma.position + 1, toks.end_pos) + ranges.append(toks) + return ranges + + def parse_loop_statement(self, toks: TokenRange) -> LoopStatement: + return LoopStatement(body=self.parse_compound_statement(toks.consume("loop"))) + + def parse_choose_statement(self, toks: TokenRange) -> ChooseStatement: + return ChooseStatement(body=self.parse_compound_statement(toks.consume("choose"))) + + def parse_when_statement(self, toks: TokenRange) -> WhenStatement: + expr = ( + toks.consume("when") + .skipWhile(lambda t: t.is_whitespace) + .first() + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + .skipWhile(lambda t: t.is_whitespace) + ) + return WhenStatement( + wait=self.parse_wait_statement(expr), + body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + ) + + def parse_state_declaration(self, toks: TokenRange) -> StateDeclarationStatement: + toks = toks.consume("state").Revskip_while(lambda t: t.value == ";") + return StateDeclarationStatement(decl=self.parse_var_declaration(toks)) + + def parse_return_statement(self, toks: TokenRange) -> ReturnStatement: + toks = toks.consume("return").Revskip_while(lambda t: t.value == ";") + return ReturnStatement(expression=self.str(self.normalize_whitespace(toks))) + + def parse_throw_statement(self, toks: TokenRange) -> ThrowStatement: + toks = toks.consume("throw").Revskip_while(lambda t: t.value == ";") + return ThrowStatement(expression=self.str(self.normalize_whitespace(toks))) + + def parse_wait_statement(self, toks: TokenRange) -> WaitStatement: + ws = WaitStatement() + ws.first_source_line = toks.first().source_line + if toks.first().value == "state": + ws.result_is_state = True + toks = toks.consume("state") + initializer = None + if toks.first().value in ("wait", "waitNext"): + initializer = toks.Revskip_while(lambda t: t.value == ";") + ws.result = VarDeclaration( + name="_", + type="Void", + initializer="", + initializer_constructor_syntax=False, + ) + else: + name, typeRange, initializer, constructorSyntax = self.parse_declaration( + toks.Revskip_while(lambda t: t.value == ";") + ) + type_str = self.str(self.normalize_whitespace(typeRange)) + if type_str == "Void": + raise ActorCompilerError( + ws.first_source_line, + "Assigning the result of a Void wait is not allowed. Just use a standalone wait statement.", + ) + ws.result = VarDeclaration( + name=name.value, + type=self.str(self.normalize_whitespace(typeRange)), + initializer="", + initializer_constructor_syntax=False, + ) + if initializer is None: + raise ActorCompilerError( + ws.first_source_line, + "Wait statement must be a declaration or standalone statement", + ) + waitParams = ( + initializer.skipWhile(lambda t: t.is_whitespace) + .consume( + "Statement contains a wait, but is not a valid wait statement or a supported compound statement.1", + lambda t: True if t.value in ("wait", "waitNext") else False, + ) + .skipWhile(lambda t: t.is_whitespace) + .first() + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(initializer) + ) + if ( + not self.range(waitParams.end_pos, initializer.end_pos) + .consume(")") + .all_match(lambda t: t.is_whitespace) + ): + raise ActorCompilerError( + toks.first().source_line, + "Statement contains a wait, but is not a valid wait statement or a supported compound statement.2", + ) + ws.future_expression = self.str(self.normalize_whitespace(waitParams)) + ws.is_wait_next = "waitNext" in [t.value for t in initializer] + return ws + + def parse_while_statement(self, toks: TokenRange) -> WhileStatement: + expr = ( + toks.consume("while") + .first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + return WhileStatement( + expression=self.str(self.normalize_whitespace(expr)), + body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + ) + + def parse_for_statement(self, toks: TokenRange) -> Statement: + head = ( + toks.consume("for") + .first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + delim = [ + t + for t in head + if t.paren_depth == head.first().paren_depth + and t.brace_depth == head.first().brace_depth + and t.value == ";" + ] + if len(delim) == 2: + init = self.range(head.begin_pos, delim[0].position) + cond = self.range(delim[0].position + 1, delim[1].position) + next_expr = self.range(delim[1].position + 1, head.end_pos) + body = self.range(head.end_pos + 1, toks.end_pos) + return ForStatement( + init_expression=self.str(self.normalize_whitespace(init)), + cond_expression=self.str(self.normalize_whitespace(cond)), + next_expression=self.str(self.normalize_whitespace(next_expr)), + body=self.parse_compound_statement(body), + ) + delim = [ + t + for t in head + if t.paren_depth == head.first().paren_depth + and t.brace_depth == head.first().brace_depth + and t.value == ":" + ] + if len(delim) != 1: + raise ActorCompilerError( + head.first().source_line, + "for statement must be 3-arg style or c++11 2-arg style", + ) + return RangeForStatement( + range_expression=self.str( + self.normalize_whitespace( + self.range(delim[0].position + 1, head.end_pos).skipWhile( + lambda t: t.is_whitespace + ) + ) + ), + range_decl=self.str( + self.normalize_whitespace( + self.range(head.begin_pos, delim[0].position - 1).skipWhile( + lambda t: t.is_whitespace + ) + ) + ), + body=self.parse_compound_statement(self.range(head.end_pos + 1, toks.end_pos)), + ) + + def parse_if_statement(self, toks: TokenRange) -> IfStatement: + toks = toks.consume("if").skipWhile(lambda t: t.is_whitespace) + constexpr = toks.first().value == "constexpr" + if constexpr: + toks = toks.consume("constexpr").skipWhile(lambda t: t.is_whitespace) + expr = ( + toks.first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + return IfStatement( + expression=self.str(self.normalize_whitespace(expr)), + constexpr=constexpr, + if_body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + ) + + def parse_else_statement(self, toks: TokenRange, prev: Statement) -> None: + if_stmt = prev + while isinstance(if_stmt, IfStatement) and if_stmt.else_body is not None: + if_stmt = if_stmt.else_body + if not isinstance(if_stmt, IfStatement): + raise ActorCompilerError( + toks.first().source_line, "else without matching if" + ) + if_stmt.else_body = self.parse_compound_statement(toks.consume("else")) + + def parse_try_statement(self, toks: TokenRange) -> TryStatement: + return TryStatement( + try_body=self.parse_compound_statement(toks.consume("try")), + catches=[], + ) + + def parse_catch_statement(self, toks: TokenRange, prev: Statement) -> None: + if not isinstance(prev, TryStatement): + raise ActorCompilerError( + toks.first().source_line, "catch without matching try" + ) + expr = ( + toks.consume("catch") + .first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + prev.catches.append( + TryStatement.Catch( + expression=self.str(self.normalize_whitespace(expr)), + body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + first_source_line=expr.first().source_line, + ) + ) + + illegal_keywords = {"goto", "do", "finally", "__if_exists", "__if_not_exists"} + + def parse_compound_statement(self, toks: TokenRange) -> Statement: + nonWhitespace = lambda t: not t.is_whitespace + first = toks.first(nonWhitespace) + if first.value == "{": + inBraces = first.get_matching_range_in(toks) + if ( + not self.range(inBraces.end_pos, toks.end_pos) + .consume("}") + .all_match(lambda t: t.is_whitespace) + ): + raise ActorCompilerError( + inBraces.last().source_line, + "Unexpected tokens after compound statement", + ) + return self.parse_code_block(inBraces) + statements: List[Statement] = [] + self.parse_statement(toks.skip(1), statements) + return statements[0] + + def parse_code_block(self, toks: TokenRange) -> CodeBlock: + statements: List[Statement] = [] + while True: + delim = next( + ( + t + for t in toks + if t.paren_depth == toks.first().paren_depth + and t.brace_depth == toks.first().brace_depth + and t.value in (";", "}") + ), + None, + ) + if delim is None: + break + self.parse_statement(self.range(toks.begin_pos, delim.position + 1), statements) + toks = self.range(delim.position + 1, toks.end_pos) + if not toks.all_match(lambda t: t.is_whitespace): + raise ActorCompilerError( + toks.first(lambda t: not t.is_whitespace).source_line, + "Trailing unterminated statement in code block", + ) + return CodeBlock(statements=statements) + + def parse_statement(self, toks: TokenRange, statements: List[Statement]) -> None: + nonWhitespace = lambda t: not t.is_whitespace + toks = toks.skipWhile(lambda t: t.is_whitespace) + + def Add(stmt: Statement) -> None: + stmt.first_source_line = toks.first().source_line + statements.append(stmt) + + first_val = toks.first().value + if first_val == "loop": + Add(self.parse_loop_statement(toks)) + elif first_val == "while": + Add(self.parse_while_statement(toks)) + elif first_val == "for": + Add(self.parse_for_statement(toks)) + elif first_val == "break": + Add(BreakStatement()) + elif first_val == "continue": + Add(ContinueStatement()) + elif first_val == "return": + Add(self.parse_return_statement(toks)) + elif first_val == "{": + Add(self.parse_compound_statement(toks)) + elif first_val == "if": + Add(self.parse_if_statement(toks)) + elif first_val == "else": + self.parse_else_statement(toks, statements[-1]) + elif first_val == "choose": + Add(self.parse_choose_statement(toks)) + elif first_val == "when": + Add(self.parse_when_statement(toks)) + elif first_val == "try": + Add(self.parse_try_statement(toks)) + elif first_val == "catch": + self.parse_catch_statement(toks, statements[-1]) + elif first_val == "throw": + Add(self.parse_throw_statement(toks)) + else: + if first_val in self.illegal_keywords: + raise ActorCompilerError( + toks.first().source_line, + f"Statement '{first_val}' not supported in actors.", + ) + if any(t.value in ("wait", "waitNext") for t in toks): + Add(self.parse_wait_statement(toks)) + elif first_val == "state": + Add(self.parse_state_declaration(toks)) + elif first_val == "switch" and any(t.value == "return" for t in toks): + raise ActorCompilerError( + toks.first().source_line, + "Unsupported compound statement containing return.", + ) + elif first_val.startswith("#"): + raise ActorCompilerError( + toks.first().source_line, + f'Found "{first_val}". Preprocessor directives are not supported within ACTORs', + ) + else: + cleaned = toks.Revskip_while(lambda t: t.value == ";") + if any(nonWhitespace(t) for t in cleaned): + Add( + PlainOldCodeStatement( + code=self.str(self.normalize_whitespace(cleaned)) + ";" + ) + ) + + def parse_class_context(self, toks: TokenRange) -> Tuple[bool, str]: + name = "" + if toks.begin_pos == toks.end_pos: + return False, name + + nonWhitespace = lambda t: not t.is_whitespace + while True: + first = toks.first(nonWhitespace) + if first.value == "[": + contents = first.get_matching_range_in(toks) + toks = self.range(contents.end_pos + 1, toks.end_pos) + elif first.value == "alignas": + toks = self.range(first.position + 1, toks.end_pos) + first = toks.first(nonWhitespace) + first.ensure("Expected ( after alignas", lambda t: t.value == "(") + contents = first.get_matching_range_in(toks) + toks = self.range(contents.end_pos + 1, toks.end_pos) + else: + break + + first = toks.first(nonWhitespace) + if not self.identifier_pattern.match(first.value): + return False, name + + while True: + first.ensure( + "Expected identifier", lambda t: self.identifier_pattern.match(t.value) + ) + name += first.value + toks = self.range(first.position + 1, toks.end_pos) + next_token = toks.first(nonWhitespace) + if next_token.value == "::": + name += "::" + toks = toks.skipWhile(lambda t: t.is_whitespace).skip(1) + else: + break + first = toks.first(nonWhitespace) + + toks = toks.skipWhile( + lambda t: t.is_whitespace or t.value in ("final", "explicit") + ) + first = toks.first(nonWhitespace) + if first.value in (":", "{"): + return True, name + return False, "" + + def str(self, tokens: Iterable[Token]) -> str: + return "".join(tok.value for tok in tokens) diff --git a/flow/actorcompiler_py/compare_actor_output.py b/flow/actorcompiler_py/compare_actor_output.py new file mode 100644 index 00000000000..e4208ebb4c4 --- /dev/null +++ b/flow/actorcompiler_py/compare_actor_output.py @@ -0,0 +1,60 @@ +import argparse +import sys +import re +import difflib +from pathlib import Path + +def compare_outputs(file1_path: Path, file2_path: Path) -> bool: + try: + file1_content = file1_path.read_text() + file2_content = file2_path.read_text() + + if file1_content == file2_content: + return True + + # Normalize #line directives - replace the filename part with a constant + # Pattern: #line "filename" + line_directive_pattern = re.compile(r'(#line\s+\d+\s+")[^"]*(")') + + file1_normalized = line_directive_pattern.sub( + r"\1NORMALIZED_PATH\2", file1_content + ) + file2_normalized = line_directive_pattern.sub( + r"\1NORMALIZED_PATH\2", file2_content + ) + + if file1_normalized == file2_normalized: + return True + + # Generate detailed diff of normalized content + file1_lines = file1_normalized.splitlines(keepends=True) + file2_lines = file2_normalized.splitlines(keepends=True) + + diff = difflib.unified_diff( + file1_lines, + file2_lines, + fromfile=f"{file1_path} (normalized)", + tofile=f"{file2_path} (normalized)", + lineterm="", + ) + + diff_text = "".join(diff) + print(diff_text, file=sys.stderr) + return False + + except Exception as e: + print(f"Error comparing files: {e}", file=sys.stderr) + return False + +def main(): + parser = argparse.ArgumentParser(description="Compare actor compiler outputs") + parser.add_argument("file1", type=Path, help="First file to compare") + parser.add_argument("file2", type=Path, help="Second file to compare") + args = parser.parse_args() + + if not compare_outputs(args.file1, args.file2): + sys.exit(1) + +if __name__ == "__main__": + main() + diff --git a/flow/actorcompiler_py/errors.py b/flow/actorcompiler_py/errors.py new file mode 100644 index 00000000000..6ad783a7246 --- /dev/null +++ b/flow/actorcompiler_py/errors.py @@ -0,0 +1,15 @@ +from __future__ import annotations + + +class ActorCompilerError(Exception): + """Exception raised for parser or compiler errors with source locations.""" + + def __init__(self, source_line: int, message: str, *args: object) -> None: + if args: + message = message.format(*args) + super().__init__(message) + self.source_line = source_line + + def __str__(self) -> str: + return f"{super().__str__()} (line {self.source_line})" + diff --git a/flow/actorcompiler_py/parse_tree.py b/flow/actorcompiler_py/parse_tree.py new file mode 100644 index 00000000000..18d3cee0a11 --- /dev/null +++ b/flow/actorcompiler_py/parse_tree.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import List, Optional, Sequence + + +@dataclass +class VarDeclaration: + type: str = "" + name: str = "" + initializer: Optional[str] = "" + initializer_constructor_syntax: bool = False + + +@dataclass +class Statement(ABC): + first_source_line: int = 0 + + @abstractmethod + def contains_wait(self) -> bool: + pass + + +@dataclass +class CodeBlock(Statement): + statements: Sequence[Statement] = field(default_factory=list) + + def __str__(self) -> str: + joined = "\n".join(str(stmt) for stmt in self.statements) + return f"CodeBlock\n{joined}\nEndCodeBlock" + + def contains_wait(self) -> bool: + return any(stmt.contains_wait() for stmt in self.statements) + + +@dataclass +class PlainOldCodeStatement(Statement): + code: str = "" + + def __str__(self) -> str: + return self.code + + def contains_wait(self) -> bool: + return False + + +@dataclass +class StateDeclarationStatement(Statement): + decl: VarDeclaration = field(default_factory=VarDeclaration) + + def __str__(self) -> str: + if self.decl.initializer_constructor_syntax: + return f"State {self.decl.type} {self.decl.name}({self.decl.initializer});" + return f"State {self.decl.type} {self.decl.name} = {self.decl.initializer};" + + def contains_wait(self) -> bool: + return False + + +@dataclass +class WhileStatement(Statement): + expression: str = "" + body: Statement = field(default_factory=CodeBlock) + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class ForStatement(Statement): + init_expression: str = "" + cond_expression: str = "" + next_expression: str = "" + body: Statement = field(default_factory=CodeBlock) + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class RangeForStatement(Statement): + range_expression: str = "" + range_decl: str = "" + body: Statement = field(default_factory=CodeBlock) + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class LoopStatement(Statement): + body: Statement = field(default_factory=CodeBlock) + + def __str__(self) -> str: + return f"Loop {self.body}" + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class BreakStatement(Statement): + def contains_wait(self) -> bool: + return False + + +@dataclass +class ContinueStatement(Statement): + def contains_wait(self) -> bool: + return False + + +@dataclass +class IfStatement(Statement): + expression: str = "" + constexpr: bool = False + if_body: Statement = field(default_factory=CodeBlock) + else_body: Optional[Statement] = None + + def contains_wait(self) -> bool: + return self.if_body.contains_wait() or ( + self.else_body is not None and self.else_body.contains_wait() + ) + + +@dataclass +class ReturnStatement(Statement): + expression: str = "" + + def __str__(self) -> str: + return f"Return {self.expression}" + + def contains_wait(self) -> bool: + return False + + +@dataclass +class WaitStatement(Statement): + result: VarDeclaration = field(default_factory=VarDeclaration) + future_expression: str = "" + result_is_state: bool = False + is_wait_next: bool = False + + def __str__(self) -> str: + return f"Wait {self.result.type} {self.result.name} <- {self.future_expression} ({'state' if self.result_is_state else 'local'})" + + def contains_wait(self) -> bool: + return True + + +@dataclass +class ChooseStatement(Statement): + body: Statement = field(default_factory=CodeBlock) + + def __str__(self) -> str: + return f"Choose {self.body}" + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class WhenStatement(Statement): + wait: WaitStatement = field(default_factory=WaitStatement) + body: Optional[Statement] = None + + def __str__(self) -> str: + return f"When ({self.wait}) {self.body}" + + def contains_wait(self) -> bool: + return True + + +@dataclass +class TryStatement(Statement): + @dataclass + class Catch: + expression: str = "" + body: Statement = field(default_factory=CodeBlock) + first_source_line: int = 0 + + try_body: Statement = field(default_factory=CodeBlock) + catches: List["TryStatement.Catch"] = field(default_factory=list) + + def contains_wait(self) -> bool: + if self.try_body.contains_wait(): + return True + return any(c.body.contains_wait() for c in self.catches) + + +@dataclass +class ThrowStatement(Statement): + expression: str = "" + + def contains_wait(self) -> bool: + return False + + +@dataclass +class Declaration: + type: str = "" + name: str = "" + comment: str = "" + + +@dataclass +class Actor: + attributes: List[str] = field(default_factory=list) + return_type: Optional[str] = None + name: str = "" + enclosing_class: Optional[str] = None + parameters: Sequence[VarDeclaration] = field(default_factory=list) + template_formals: Optional[Sequence[VarDeclaration]] = None + body: CodeBlock = field(default_factory=CodeBlock) + source_line: int = 0 + is_static: bool = False + _is_uncancellable: bool = False + test_case_parameters: Optional[str] = None + name_space: Optional[str] = None + is_forward_declaration: bool = False + is_test_case: bool = False + + def is_cancellable(self) -> bool: + return self.return_type is not None and not self._is_uncancellable + + def set_uncancellable(self) -> None: + self._is_uncancellable = True + + +@dataclass +class Descr: + name: str = "" + super_class_list: Optional[str] = None + body: List[Declaration] = field(default_factory=list)