From 95ba91d6ed7c5b71b774e27c30092815238483a7 Mon Sep 17 00:00:00 2001 From: Hendrik Hofstadt Date: Mon, 17 Nov 2025 11:30:35 +0000 Subject: [PATCH 1/2] Port actorcompiler to python --- cmake/CompileActorCompiler.cmake | 51 +- cmake/FlowCommands.cmake | 27 +- flow/actorcompiler/ActorCompiler.cs | 1476 --------------- flow/actorcompiler/ActorCompiler.targets | 54 - flow/actorcompiler/ActorCompiler.xml | 62 - flow/actorcompiler/ActorParser.cs | 1178 ------------ flow/actorcompiler/ParseTree.cs | 252 --- flow/actorcompiler/Program.cs | 104 -- flow/actorcompiler/Properties/AssemblyInfo.cs | 56 - flow/actorcompiler/__init__.py | 33 + flow/actorcompiler/__main__.py | 10 + flow/actorcompiler/actor_compiler.py | 1583 +++++++++++++++++ flow/actorcompiler/actor_parser.py | 1134 ++++++++++++ flow/actorcompiler/actorcompiler.csproj | 10 - flow/actorcompiler/main.py | 97 + flow/actorcompiler/parse_tree.py | 219 +++ flow/actorcompiler/usertype.dat | 8 - 17 files changed, 3093 insertions(+), 3261 deletions(-) delete mode 100644 flow/actorcompiler/ActorCompiler.cs delete mode 100644 flow/actorcompiler/ActorCompiler.targets delete mode 100644 flow/actorcompiler/ActorCompiler.xml delete mode 100644 flow/actorcompiler/ActorParser.cs delete mode 100644 flow/actorcompiler/ParseTree.cs delete mode 100644 flow/actorcompiler/Program.cs delete mode 100644 flow/actorcompiler/Properties/AssemblyInfo.cs create mode 100644 flow/actorcompiler/__init__.py create mode 100644 flow/actorcompiler/__main__.py create mode 100644 flow/actorcompiler/actor_compiler.py create mode 100644 flow/actorcompiler/actor_parser.py delete mode 100644 flow/actorcompiler/actorcompiler.csproj create mode 100644 flow/actorcompiler/main.py create mode 100644 flow/actorcompiler/parse_tree.py delete mode 100644 flow/actorcompiler/usertype.dat diff --git a/cmake/CompileActorCompiler.cmake b/cmake/CompileActorCompiler.cmake index 0b0c1e6d345..5443a82e33c 100644 --- a/cmake/CompileActorCompiler.cmake +++ b/cmake/CompileActorCompiler.cmake @@ -1,41 +1,14 @@ -set(ACTORCOMPILER_CSPROJ - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actorcompiler.csproj) +find_package(Python3 REQUIRED COMPONENTS Interpreter) + set(ACTORCOMPILER_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorCompiler.cs - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorParser.cs - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ParseTree.cs - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Program.cs - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Properties/AssemblyInfo.cs) -set(ACTOR_COMPILER_REFERENCES - "-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml" -) + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/__init__.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/__main__.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/main.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actor_parser.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actor_compiler.py) + +add_custom_target(actorcompiler DEPENDS ${ACTORCOMPILER_SRCS}) -if(WIN32) - add_executable(actorcompiler ${ACTORCOMPILER_SRCS}) - target_compile_options(actorcompiler PRIVATE "/langversion:6") - set_property( - TARGET actorcompiler - PROPERTY VS_DOTNET_REFERENCES - "System" - "System.Core" - "System.Xml.Linq" - "System.Data.DataSetExtensions" - "Microsoft.CSharp" - "System.Data" - "System.Xml") -elseif(CSHARP_USE_MONO) - add_custom_command( - OUTPUT actorcompiler.exe - COMMAND ${CSHARP_COMPILER_EXECUTABLE} ARGS ${ACTOR_COMPILER_REFERENCES} - ${ACTORCOMPILER_SRCS} "-target:exe" "-out:actorcompiler.exe" - DEPENDS ${ACTORCOMPILER_SRCS} - COMMENT "Compile actor compiler" - VERBATIM) - add_custom_target(actorcompiler - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe) - set(actor_exe "${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe") -else() - dotnet_build(${ACTORCOMPILER_CSPROJ} SOURCE ${ACTORCOMPILER_SRCS}) - set(actor_exe "${actorcompiler_EXECUTABLE_PATH}") - message(STATUS "Actor compiler path: ${actor_exe}") -endif() +set(ACTORCOMPILER_COMMAND + ${Python3_EXECUTABLE} -m flow.actorcompiler + CACHE INTERNAL "Command to run the actor compiler") diff --git a/cmake/FlowCommands.cmake b/cmake/FlowCommands.cmake index dadbab42fdc..e0f77908fbe 100644 --- a/cmake/FlowCommands.cmake +++ b/cmake/FlowCommands.cmake @@ -263,28 +263,11 @@ function(add_flow_target) endforeach() list(APPEND generated_files ${out_file}) - if(WIN32) - add_custom_command( - OUTPUT "${out_file}" - COMMAND $ "${in_file}" "${out_file}" - ${actor_compiler_flags} - DEPENDS "${in_file}" actorcompiler - COMMENT "Compile actor: ${src}") - elseif(CSHARP_USE_MONO) - add_custom_command( - OUTPUT "${out_file}" - COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${in_file}" "${out_file}" - ${actor_compiler_flags} > /dev/null - DEPENDS "${in_file}" actorcompiler - COMMENT "Compile actor: ${src}") - else() - add_custom_command( - OUTPUT "${out_file}" - COMMAND ${actor_exe} "${in_file}" "${out_file}" - ${actor_compiler_flags} > /dev/null - DEPENDS "${in_file}" actorcompiler - COMMENT "Compile actor: ${src}") - endif() + add_custom_command(OUTPUT "${out_file}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CMAKE_SOURCE_DIR}" + ${ACTORCOMPILER_COMMAND} "${in_file}" "${out_file}" ${actor_compiler_flags} + DEPENDS "${in_file}" actorcompiler + COMMENT "Compile actor: ${src}") endif() endforeach() if(PASS_COMPILATION_UNIT) diff --git a/flow/actorcompiler/ActorCompiler.cs b/flow/actorcompiler/ActorCompiler.cs deleted file mode 100644 index ab906d47d5d..00000000000 --- a/flow/actorcompiler/ActorCompiler.cs +++ /dev/null @@ -1,1476 +0,0 @@ -/* - * ActorCompiler.cs - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using System.Security.Cryptography; - -namespace actorcompiler -{ - class TypeSwitch - { - object value; - R result; - bool ok = false; - public TypeSwitch(object t) { this.value = t; } - public TypeSwitch Case(Func f) - where T : class - { - if (!ok) - { - var t = value as T; - if (t != null) - { - result = f(t); - ok = true; - } - } - return this; - } - public R Return() - { - if (!ok) throw new Exception("Typeswitch didn't match."); - return result; - } - } - - class Context - { - public Function target; - public Function next; - public Function breakF; - public Function continueF; - public Function catchFErr; // Catch function taking Error - public int tryLoopDepth = 0; // The number of (loopDepth-increasing) loops entered inside the innermost try (thus, that will be exited by a throw) - - public void unreachable() { target = null; } - - // Usually we just change the target of a context - public Context WithTarget(Function newTarget) { return new Context { target = newTarget, breakF = breakF, continueF = continueF, next = null, catchFErr = catchFErr, tryLoopDepth = tryLoopDepth }; } - - // When entering a loop, we have to provide new break and continue functions - public Context LoopContext(Function newTarget, Function breakF, Function continueF, int deltaLoopDepth) { return new Context { target = newTarget, breakF = breakF, continueF = continueF, next = null, catchFErr = catchFErr, tryLoopDepth = tryLoopDepth + deltaLoopDepth }; } - - public Context WithCatch(Function newCatchFErr) { return new Context { target = target, breakF = breakF, continueF = continueF, next = null, catchFErr = newCatchFErr, tryLoopDepth = 0 }; } - - public Context Clone() { return new Context { target = target, next = next, breakF = breakF, continueF = continueF, catchFErr = catchFErr, tryLoopDepth = tryLoopDepth }; } - }; - - class Function - { - public string name; - public string returnType; - public string[] formalParameters; - public bool endIsUnreachable = false; - public string exceptionParameterIs = null; - public bool publicName = false; - public string specifiers; - string indentation; - StreamWriter body; - public bool wasCalled { get; protected set; } - public Function overload = null; - - public Function() - { - body = new StreamWriter(new MemoryStream()); - } - - public void setOverload(Function overload) { - this.overload = overload; - } - - public Function popOverload() { - Function result = this.overload; - this.overload = null; - return result; - } - - public void addOverload(params string[] formalParameters) { - setOverload( - new Function { - name = name, - returnType = returnType, - endIsUnreachable = endIsUnreachable, - formalParameters = formalParameters, - indentation = indentation - } - ); - } - - public void Indent(int change) - { - for(int i=0; i0, run the actor beginning at point P until - * (1) it waits, in which case set an appropriate callback and return 0, or - * (2) it returns, in which case destroy the actor and return 0, or - * (3) it reaches the bottom of the Nth innermost loop containing P, in which case - * return max(0, the given loopDepth - N) (N=0 for the innermost loop, N=1 for the next innermost, etc) - * - * Examples: - * Source: - * loop - * [P] - * loop - * [P'] - * loop - * [P''] - * break - * [Q'] - * break - * [Q] - * - * fP(1) should execute everything from [P] to [Q] and then return 1 (since [Q] is at the bottom of the 0th innermost loop containing [P]) - * fP'(2) should execute everything from [P'] to [Q] and then return 1 (since [Q] is at the bottom of the 1st innermost loop containing [P']) - * fP''(3) should execute everything from [P''] to [Q] and then return 1 (since [Q] is at the bottom of the 2nd innermost loop containing [P'']) - * fQ'(2) should execute everything from [Q'] to [Q] and then return 1 (since [Q] is at the bottom of the 1st innermost loop containing [Q']) - * fQ(1) should return 1 (since [Q] is at the bottom of the 0th innermost loop containing [Q]) - */ - }; - - class LiteralBreak : Function - { - public LiteralBreak() { name = "break!"; } - public override string call(params string[] parameters) - { - wasCalled = true; - if (parameters.Length != 0) throw new Exception("LiteralBreak called with parameters!"); - return "break"; - } - }; - - class LiteralContinue : Function - { - public LiteralContinue() { name = "continue!"; } - public override string call(params string[] parameters) - { - wasCalled = true; - if (parameters.Length != 0) throw new Exception("LiteralContinue called with parameters!"); - return "continue"; - } - }; - - class StateVar : VarDeclaration - { - public int SourceLine; - }; - - class CallbackVar : StateVar - { - public int CallbackGroup; - } - - class DescrCompiler - { - Descr descr; - string memberIndentStr; - - public DescrCompiler(Descr descr, int braceDepth) - { - this.descr = descr; - this.memberIndentStr = new string('\t', braceDepth); - } - public void Write(TextWriter writer, out int lines) - { - lines = 0; - - writer.WriteLine(memberIndentStr + "template<> struct Descriptor {{", descr.name); - writer.WriteLine(memberIndentStr + "\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", descr.name); - writer.WriteLine(memberIndentStr + "\ttypedef {0} type;", descr.name); - lines += 3; - - foreach (var dec in descr.body) - { - writer.WriteLine(memberIndentStr + "\tstruct {0}Descriptor {{", dec.name); - writer.WriteLine(memberIndentStr + "\t\tstatic StringRef name() {{ return \"{0}\"_sr; }}", dec.name); - writer.WriteLine(memberIndentStr + "\t\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", dec.type); - writer.WriteLine(memberIndentStr + "\t\tstatic StringRef comment() {{ return \"{0}\"_sr; }}", dec.comment); - writer.WriteLine(memberIndentStr + "\t\ttypedef {0} type;", dec.type); - writer.WriteLine(memberIndentStr + "\t\tstatic inline type get({0}& from);", descr.name); - writer.WriteLine(memberIndentStr + "\t};"); - lines += 7; - } - - writer.Write(memberIndentStr + "\ttypedef std::tuple<"); - bool FirstDesc = true; - foreach (var dec in descr.body) - { - if (!FirstDesc) - writer.Write(","); - writer.Write("{0}Descriptor", dec.name); - FirstDesc = false; - } - writer.Write("> fields;\n"); - writer.WriteLine(memberIndentStr + "\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type field_indexes;"); - writer.WriteLine(memberIndentStr + "};"); - if(descr.superClassList != null) - writer.WriteLine(memberIndentStr + "struct {0} : {1} {{", descr.name, descr.superClassList); - else - writer.WriteLine(memberIndentStr + "struct {0} {{", descr.name); - lines += 4; - - foreach (var dec in descr.body) - { - writer.WriteLine(memberIndentStr + "\t{0} {1}; //{2}", dec.type, dec.name, dec.comment); - lines++; - } - - writer.WriteLine(memberIndentStr + "};"); - lines++; - - foreach (var dec in descr.body) - { - writer.WriteLine(memberIndentStr + "{0} Descriptor<{1}>::{2}Descriptor::get({1}& from) {{ return from.{2}; }}", dec.type, descr.name, dec.name); - lines++; - } - - } - } - - class ActorCompiler - { - Actor actor; - string className, fullClassName, stateClassName; - string sourceFile; - List state; - List callbacks = new List(); - bool isTopLevel; - const string loopDepth0 = "int loopDepth=0"; - const string loopDepth = "int loopDepth"; - const int codeIndent = +2; - const string memberIndentStr = "\t"; - static HashSet usedClassNames = new HashSet(); - bool LineNumbersEnabled; - int chooseGroups = 0, whenCount = 0; - string This; - bool generateProbes; - public Dictionary<(ulong, ulong), string> uidObjects { get; private set; } - - public ActorCompiler(Actor actor, string sourceFile, bool isTopLevel, bool lineNumbersEnabled, bool generateProbes) - { - this.actor = actor; - this.sourceFile = sourceFile; - this.isTopLevel = isTopLevel; - this.LineNumbersEnabled = lineNumbersEnabled; - this.generateProbes = generateProbes; - this.uidObjects = new Dictionary<(ulong, ulong), string>(); - - FindState(); - } - - private ulong ByteToLong(byte[] bytes) { - // NOTE: Always assume big endian. - ulong result = 0; - foreach(var b in bytes) { - result += b; - result <<= 8; - } - return result; - } - - // Generates the identifier for the ACTOR - private Tuple GetUidFromString(string str) { - byte[] sha256Hash = SHA256.Create().ComputeHash(Encoding.UTF8.GetBytes(str)); - byte[] first = sha256Hash.Take(8).ToArray(); - byte[] second = sha256Hash.Skip(8).Take(8).ToArray(); - return new Tuple(ByteToLong(first), ByteToLong(second)); - } - - // Writes the function that returns the Actor object - private void WriteActorFunction(TextWriter writer, string fullReturnType) { - WriteTemplate(writer); - LineNumber(writer, actor.SourceLine); - foreach (string attribute in actor.attributes) { - writer.Write(attribute + " "); - } - if (actor.isStatic) writer.Write("static "); - writer.WriteLine("{0} {3}{1}( {2} ) {{", fullReturnType, actor.name, string.Join(", ", ParameterList()), actor.nameSpace==null ? "" : actor.nameSpace + "::"); - LineNumber(writer, actor.SourceLine); - - string newActor = string.Format("new {0}({1})", - fullClassName, - string.Join(", ", actor.parameters.Select(p => p.name).ToArray())); - - if (actor.returnType != null) - writer.WriteLine("\treturn Future<{1}>({0});", newActor, actor.returnType); - else - writer.WriteLine("\t{0};", newActor); - writer.WriteLine("}"); - } - - // Writes the class of the Actor object - private void WriteActorClass(TextWriter writer, string fullStateClassName, Function body) { - // The final actor class mixes in the State class, the Actor base class and all callback classes - writer.WriteLine("// This generated class is to be used only via {0}()", actor.name); - WriteTemplate(writer); - LineNumber(writer, actor.SourceLine); - - string callback_base_classes = string.Join(", ", callbacks.Select(c=>string.Format("public {0}", c.type))); - if (callback_base_classes != "") callback_base_classes += ", "; - writer.WriteLine("class {0} final : public Actor<{2}>, {3}public FastAllocated<{1}>, public {4} {{", - className, - fullClassName, - actor.returnType == null ? "void" : actor.returnType, - callback_base_classes, - fullStateClassName - ); - writer.WriteLine("public:"); - writer.WriteLine("\tusing FastAllocated<{0}>::operator new;", fullClassName); - writer.WriteLine("\tusing FastAllocated<{0}>::operator delete;", fullClassName); - - var actorIdentifierKey = this.sourceFile + ":" + this.actor.name; - var actorIdentifier = GetUidFromString(actorIdentifierKey); - uidObjects.Add((actorIdentifier.Item1, actorIdentifier.Item2), actorIdentifierKey); - // NOTE UL is required as a u64 postfix for large integers, otherwise Clang would complain - writer.WriteLine("\tstatic constexpr ActorIdentifier __actorIdentifier = UID({0}UL, {1}UL);", actorIdentifier.Item1, actorIdentifier.Item2); - writer.WriteLine("\tActiveActorHelper activeActorHelper;"); - - writer.WriteLine("#pragma clang diagnostic push"); - writer.WriteLine("#pragma clang diagnostic ignored \"-Wdelete-non-virtual-dtor\""); - if (actor.returnType != null) - writer.WriteLine(@" void destroy() override {{ - activeActorHelper.~ActiveActorHelper(); - static_cast*>(this)->~Actor(); - operator delete(this); - }}", actor.returnType); - else - writer.WriteLine(@" void destroy() {{ - activeActorHelper.~ActiveActorHelper(); - static_cast*>(this)->~Actor(); - operator delete(this); - }}"); - writer.WriteLine("#pragma clang diagnostic pop"); - - foreach (var cb in callbacks) - writer.WriteLine("friend struct {0};", cb.type); - - LineNumber(writer, actor.SourceLine); - WriteConstructor(body, writer, fullStateClassName); - WriteCancelFunc(writer); - writer.WriteLine("};"); - - } - - public void Write(TextWriter writer) - { - string fullReturnType = - actor.returnType != null ? string.Format("Future<{0}>", actor.returnType) - : "void"; - for (int i = 0; ; i++) - { - className = string.Format("{3}{0}{1}Actor{2}", - actor.name.Substring(0, 1).ToUpper(), - actor.name.Substring(1), - i != 0 ? i.ToString() : "", - actor.enclosingClass != null && actor.isForwardDeclaration ? actor.enclosingClass.Replace("::", "_") + "_" - : actor.nameSpace != null ? actor.nameSpace.Replace("::", "_") + "_" - : ""); - if (actor.isForwardDeclaration || usedClassNames.Add(className)) - break; - } - - // e.g. SimpleTimerActor - fullClassName = className + GetTemplateActuals(); - var actorClassFormal = new VarDeclaration { name = className, type = "class" }; - This = string.Format("static_cast<{0}*>(this)", actorClassFormal.name); - // e.g. SimpleTimerActorState - stateClassName = className + "State"; - // e.g. SimpleTimerActorState - var fullStateClassName = stateClassName + GetTemplateActuals(new VarDeclaration { type = "class", name = fullClassName }); - - if (actor.isForwardDeclaration) { - foreach (string attribute in actor.attributes) { - writer.Write(attribute + " "); - } - if (actor.isStatic) writer.Write("static "); - writer.WriteLine("{0} {3}{1}( {2} );", fullReturnType, actor.name, string.Join(", ", ParameterList()), actor.nameSpace==null ? "" : actor.nameSpace + "::"); - if (actor.enclosingClass != null) { - writer.WriteLine("template friend class {0};", stateClassName); - } - return; - } - - var body = getFunction("", "body", loopDepth0); - var bodyContext = new Context { - target = body, - catchFErr = getFunction(body.name, "Catch", "Error error", loopDepth0), - }; - - var endContext = TryCatchCompile(actor.body, bodyContext); - - if (endContext.target != null) - { - if (actor.returnType == null) - CompileStatement(new ReturnStatement { FirstSourceLine = actor.SourceLine, expression = "" }, endContext ); - else - throw new Error(actor.SourceLine, "Actor {0} fails to return a value", actor.name); - } - - if (actor.returnType != null) - { - bodyContext.catchFErr.WriteLine("this->~{0}();", stateClassName); - bodyContext.catchFErr.WriteLine("{0}->sendErrorAndDelPromiseRef(error);", This); - } - else - { - bodyContext.catchFErr.WriteLine("delete {0};", This); - } - bodyContext.catchFErr.WriteLine("loopDepth = 0;"); - - if (isTopLevel && actor.nameSpace == null) writer.WriteLine("namespace {"); - - // The "State" class contains all state and user code, to make sure that state names are accessible to user code but - // inherited members of Actor, Callback etc are not. - writer.WriteLine("// This generated class is to be used only via {0}()", actor.name); - WriteTemplate(writer, actorClassFormal); - LineNumber(writer, actor.SourceLine); - writer.WriteLine("class {0} {{", stateClassName); - writer.WriteLine("public:"); - LineNumber(writer, actor.SourceLine); - WriteStateConstructor(writer); - WriteStateDestructor(writer); - WriteFunctions(writer); - foreach (var st in state) - { - LineNumber(writer, st.SourceLine); - writer.WriteLine("\t{0} {1};", st.type, st.name); - } - writer.WriteLine("};"); - - WriteActorClass(writer, fullStateClassName, body); - - if (isTopLevel && actor.nameSpace == null) writer.WriteLine("} // namespace"); // namespace - - WriteActorFunction(writer, fullReturnType); - - if (actor.testCaseParameters != null) - { - writer.WriteLine("ACTOR_TEST_CASE({0}, {1})", actor.name, actor.testCaseParameters); - } - - Console.WriteLine("\tCompiled ACTOR {0} (line {1})", actor.name, actor.SourceLine); - } - - const string thisAddress = "reinterpret_cast(this)"; - - void ProbeEnter(Function fun, string name, int index = -1) { - if (generateProbes) { - fun.WriteLine("fdb_probe_actor_enter(\"{0}\", {1}, {2});", name, thisAddress, index); - } - var blockIdentifier = GetUidFromString(fun.name); - fun.WriteLine("#ifdef WITH_ACAC"); - fun.WriteLine("static constexpr ActorBlockIdentifier __identifier = UID({0}UL, {1}UL);", blockIdentifier.Item1, blockIdentifier.Item2); - fun.WriteLine("ActorExecutionContextHelper __helper(static_cast<{0}*>(this)->activeActorHelper.actorID, __identifier);", className); - fun.WriteLine("#endif // WITH_ACAC"); - } - - void ProbeExit(Function fun, string name, int index = -1) { - if (generateProbes) { - fun.WriteLine("fdb_probe_actor_exit(\"{0}\", {1}, {2});", name, thisAddress, index); - } - } - - void ProbeCreate(Function fun, string name) { - if (generateProbes) { - fun.WriteLine("fdb_probe_actor_create(\"{0}\", {1});", name, thisAddress); - } - } - - void ProbeDestroy(Function fun, string name) { - if (generateProbes) { - fun.WriteLine("fdb_probe_actor_destroy(\"{0}\", {1});", name, thisAddress); - } - } - - void LineNumber(TextWriter writer, int SourceLine) - { - if(SourceLine == 0) - { - throw new Exception("Internal error: Invalid source line (0)"); - } - if (LineNumbersEnabled) - writer.WriteLine("\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} \"{1}\"", SourceLine, sourceFile); - } - void LineNumber(Function writer, int SourceLine) - { - if(SourceLine == 0) - { - throw new Exception("Internal error: Invalid source line (0)"); - } - if (LineNumbersEnabled) - writer.WriteLineUnindented( string.Format("\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} \"{1}\"", SourceLine, sourceFile) ); - } - - void TryCatch(Context cx, Function catchFErr, int catchLoopDepth, Action action, bool useLoopDepth = true) - { - if (catchFErr!=null) - { - cx.target.WriteLine("try {"); - cx.target.Indent(+1); - } - action(); - if (catchFErr!=null) - { - cx.target.Indent(-1); - cx.target.WriteLine("}"); - - cx.target.WriteLine("catch (Error& error) {"); - if (useLoopDepth) - cx.target.WriteLine("\tloopDepth = {0};", catchFErr.call("error", AdjustLoopDepth(catchLoopDepth))); - else - cx.target.WriteLine("\t{0};", catchFErr.call("error", "0")); - cx.target.WriteLine("} catch (...) {"); - if (useLoopDepth) - cx.target.WriteLine("\tloopDepth = {0};", catchFErr.call("unknown_error()", AdjustLoopDepth(catchLoopDepth))); - else - cx.target.WriteLine("\t{0};", catchFErr.call("unknown_error()", "0")); - cx.target.WriteLine("}"); - } - } - Context TryCatchCompile(CodeBlock block, Context cx) - { - TryCatch(cx, cx.catchFErr, cx.tryLoopDepth, () => { - cx = Compile(block, cx, true); - if (cx.target != null) - { - var next = getFunction(cx.target.name, "cont", loopDepth); - cx.target.WriteLine("loopDepth = {0};", next.call("loopDepth")); - cx.target = next; - cx.next = null; - } - }); - return cx; - } - - void WriteTemplate(TextWriter writer, params VarDeclaration[] extraParameters) - { - var formals = (actor.templateFormals!=null ? actor.templateFormals.AsEnumerable() : Enumerable.Empty()) - .Concat(extraParameters) - .ToArray(); - - if (formals.Length==0) return; - LineNumber(writer, actor.SourceLine); - writer.WriteLine("template <{0}>", - string.Join(", ", formals.Select( - p => string.Format("{0} {1}", p.type, p.name) - ).ToArray())); - } - string GetTemplateActuals(params VarDeclaration[] extraParameters) - { - var formals = (actor.templateFormals != null ? actor.templateFormals.AsEnumerable() : Enumerable.Empty()) - .Concat(extraParameters) - .ToArray(); - - if (formals.Length == 0) return ""; - else return "<" + - string.Join(", ", formals.Select( - p => p.name - ).ToArray()) - + ">"; - } - - bool WillContinue(Statement stmt) - { - return Flatten(stmt).Any( - st => (st is ChooseStatement || st is WaitStatement || st is TryStatement)); - } - - CodeBlock AsCodeBlock(Statement statement) - { - // SOMEDAY: Is this necessary? Maybe we should just be compiling statements? - var cb = statement as CodeBlock; - if (cb != null) return cb; - return new CodeBlock { statements = new Statement[] { statement } }; - } - - void CompileStatement(PlainOldCodeStatement stmt, Context cx) - { - LineNumber(cx.target,stmt.FirstSourceLine); - cx.target.WriteLine(stmt.code); - } - void CompileStatement(StateDeclarationStatement stmt, Context cx) - { - // if this state declaration is at the very top of the actor body - if (actor.body.statements.Select(x => x as StateDeclarationStatement).TakeWhile(x => x != null).Any(x => x == stmt)) - { - // Initialize the state in the constructor, not here - state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, name = stmt.decl.name, type = stmt.decl.type, - initializer = stmt.decl.initializer, initializerConstructorSyntax = stmt.decl.initializerConstructorSyntax } ); - } - else - { - // State variables declared elsewhere must have a default constructor - state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, name = stmt.decl.name, type = stmt.decl.type, initializer = null }); - if (stmt.decl.initializer != null) - { - LineNumber(cx.target, stmt.FirstSourceLine); - if (stmt.decl.initializerConstructorSyntax || stmt.decl.initializer=="") - cx.target.WriteLine("{0} = {1}({2});", stmt.decl.name, stmt.decl.type, stmt.decl.initializer); - else - cx.target.WriteLine("{0} = {1};", stmt.decl.name, stmt.decl.initializer); - } - } - } - - void CompileStatement(ForStatement stmt, Context cx) - { - // for( initExpression; condExpression; nextExpression ) body; - - bool noCondition = stmt.condExpression == "" || stmt.condExpression == "true" || stmt.condExpression == "1"; - - if (!WillContinue(stmt.body)) - { - // We can write this loop without anything fancy, because there are no wait statements in it - if (EmitNativeLoop(stmt.FirstSourceLine, "for(" + stmt.initExpression + ";" + stmt.condExpression + ";" + stmt.nextExpression+")", stmt.body, cx) - && noCondition ) - cx.unreachable(); - } - else - { - // First compile the initExpression - CompileStatement( new PlainOldCodeStatement {code = stmt.initExpression + ";", FirstSourceLine = stmt.FirstSourceLine}, cx ); - - // fullBody = { if (!(condExpression)) break; body; } - Statement fullBody = noCondition ? stmt.body : - new CodeBlock - { - statements = new Statement[] { - new IfStatement - { - expression = "!(" + stmt.condExpression + ")", - ifBody = new BreakStatement { FirstSourceLine = stmt.FirstSourceLine }, - FirstSourceLine = stmt.FirstSourceLine - } - }.Concat(AsCodeBlock(stmt.body).statements).ToArray(), - FirstSourceLine = stmt.FirstSourceLine - }; - - Function loopF = getFunction(cx.target.name, "loopHead", loopDepth); - Function loopBody = getFunction(cx.target.name, "loopBody", loopDepth); - Function breakF = getFunction(cx.target.name, "break", loopDepth); - Function continueF = stmt.nextExpression == "" ? loopF : getFunction(cx.target.name, "continue", loopDepth); - - // TODO: Could we use EmitNativeLoop() here? - - loopF.WriteLine("int oldLoopDepth = ++loopDepth;"); - loopF.WriteLine("while (loopDepth == oldLoopDepth) loopDepth = {0};", loopBody.call("loopDepth")); - - Function endLoop = Compile(AsCodeBlock(fullBody), cx.LoopContext( loopBody, breakF, continueF, +1 ), true).target; - if (endLoop != null && endLoop != loopBody) { - if (stmt.nextExpression != "") - CompileStatement( new PlainOldCodeStatement {code = stmt.nextExpression + ";", FirstSourceLine = stmt.FirstSourceLine}, cx.WithTarget(endLoop) ); - endLoop.WriteLine("if (loopDepth == 0) return {0};", loopF.call("0")); - } - - cx.target.WriteLine("loopDepth = {0};", loopF.call("loopDepth")); - - if (continueF != loopF && continueF.wasCalled) { - CompileStatement( new PlainOldCodeStatement {code = stmt.nextExpression + ";", FirstSourceLine = stmt.FirstSourceLine}, cx.WithTarget(continueF) ); - continueF.WriteLine("if (loopDepth == 0) return {0};", loopF.call("0")); - } - - if (breakF.wasCalled) - TryCatch(cx.WithTarget(breakF), cx.catchFErr, cx.tryLoopDepth, - () => { breakF.WriteLine("return {0};", cx.next.call("loopDepth")); }); - else - cx.unreachable(); - } - } - - Dictionary iterators = new Dictionary(); - - string getIteratorName(Context cx) - { - string name = "RangeFor" + cx.target.name + "Iterator"; - if (!iterators.ContainsKey(name)) - iterators[name] = 0; - return string.Format("{0}{1}", name, iterators[name]++); - } - - void CompileStatement(RangeForStatement stmt, Context cx) - { - // If stmt does not contain a wait statement, rewrite the original c++11 range-based for loop - // If there is a wait, we need to rewrite the loop as: - // for(a:b) c; ==> for(__iter=std::begin(b); __iter!=std::end(b); ++__iter) { a = *__iter; c; } - // where __iter is stored as a state variable - - if (WillContinue(stmt.body)) - { - StateVar container = state.FirstOrDefault(s => s.name == stmt.rangeExpression); - if (container == null) - { - throw new Error(stmt.FirstSourceLine, "container of range-based for with continuation must be a state variable"); - } - - var iter = getIteratorName(cx); - state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, name = iter, type = "decltype(std::begin(std::declval<" + container.type + ">()))", initializer = null }); - var equivalent = new ForStatement { - initExpression = iter + " = std::begin(" + stmt.rangeExpression + ")", - condExpression = iter + " != std::end(" + stmt.rangeExpression + ")", - nextExpression = "++" + iter, - FirstSourceLine = stmt.FirstSourceLine, - body = new CodeBlock - { - statements = new Statement[] { - new PlainOldCodeStatement { FirstSourceLine = stmt.FirstSourceLine, code = stmt.rangeDecl + " = *" + iter + ";" }, - stmt.body - } - } - }; - CompileStatement(equivalent, cx); - } - else - { - EmitNativeLoop(stmt.FirstSourceLine, "for( " + stmt.rangeDecl + " : " + stmt.rangeExpression + " )", stmt.body, cx); - } - } - - void CompileStatement(WhileStatement stmt, Context cx) - { - // Compile while (x) { y } as for(;x;) { y } - var equivalent = new ForStatement - { - condExpression = stmt.expression, - body = stmt.body, - FirstSourceLine = stmt.FirstSourceLine, - }; - - CompileStatement(equivalent, cx); - } - - void CompileStatement(LoopStatement stmt, Context cx) - { - // Compile loop { body } as for(;;;) { body } - var equivalent = new ForStatement - { - body = stmt.body, - FirstSourceLine = stmt.FirstSourceLine - }; - CompileStatement(equivalent, cx); - } - - // Writes out a loop in native C++ (with no continuation passing) - // Returns true if the loop is known to have no normal exit (is unreachable) - private bool EmitNativeLoop(int sourceLine, string head, Statement body, Context cx) - { - LineNumber(cx.target, sourceLine); - cx.target.WriteLine(head + " {"); - cx.target.Indent(+1); - var literalBreak = new LiteralBreak(); - Compile(AsCodeBlock(body), cx.LoopContext(cx.target, literalBreak, new LiteralContinue(), 0), true); - cx.target.Indent(-1); - cx.target.WriteLine("}"); - return !literalBreak.wasCalled; - } - - void CompileStatement(ChooseStatement stmt, Context cx) - { - int group = ++this.chooseGroups; - //string cbGroup = "ChooseGroup" + getFunction(cx.target.name,"W").name; // SOMEDAY - - var codeblock = stmt.body as CodeBlock; - if (codeblock == null) - throw new Error(stmt.FirstSourceLine, "'choose' must be followed by a compound statement."); - var choices = codeblock.statements - .OfType() - .Select( (ch,i) => new { - Stmt = ch, - Group = group, - Index = this.whenCount+i, - Body = getFunction(cx.target.name, "when", - new string[] { string.Format("{0} const& {2}{1}", ch.wait.result.type, ch.wait.result.name, ch.wait.resultIsState?"__":""), loopDepth }, - new string[] { string.Format("{0} && {2}{1}", ch.wait.result.type, ch.wait.result.name, ch.wait.resultIsState?"__":""), loopDepth } - ), - Future = string.Format("__when_expr_{0}", this.whenCount + i), - CallbackType = string.Format("{3}< {0}, {1}, {2} >", fullClassName, this.whenCount + i, ch.wait.result.type, ch.wait.isWaitNext ? "ActorSingleCallback" : "ActorCallback"), - CallbackTypeInStateClass = string.Format("{3}< {0}, {1}, {2} >", className, this.whenCount + i, ch.wait.result.type, ch.wait.isWaitNext ? "ActorSingleCallback" : "ActorCallback") - }) - .ToArray(); - this.whenCount += choices.Length; - if (choices.Length != codeblock.statements.Length) - throw new Error(codeblock.statements.First(x=>!(x is WhenStatement)).FirstSourceLine, "only 'when' statements are valid in an 'choose' block."); - - var exitFunc = getFunction("exitChoose", ""); - exitFunc.returnType = "void"; - exitFunc.WriteLine("if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", This); - foreach(var ch in choices) - exitFunc.WriteLine("{0}->{1}::remove();", This, ch.CallbackTypeInStateClass); - exitFunc.endIsUnreachable = true; - - //state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, type = "CallbackGroup", name = cbGroup, callbackCatchFErr = cx.catchFErr }); - bool reachable = false; - foreach(var ch in choices) { - callbacks.Add(new CallbackVar - { - SourceLine = ch.Stmt.FirstSourceLine, - CallbackGroup = ch.Group, - type = ch.CallbackType - }); - var r = ch.Body; - if (ch.Stmt.wait.resultIsState) - { - Function overload = r.popOverload(); - CompileStatement(new StateDeclarationStatement - { - FirstSourceLine = ch.Stmt.FirstSourceLine, - decl = new VarDeclaration { - type = ch.Stmt.wait.result.type, - name = ch.Stmt.wait.result.name, - initializer = "__" + ch.Stmt.wait.result.name, - initializerConstructorSyntax = false - } - }, cx.WithTarget(r)); - if (overload != null) - { - overload.WriteLine("{0} = std::move(__{0});", ch.Stmt.wait.result.name); - r.setOverload(overload); - } - } - if (ch.Stmt.body != null) - { - r = Compile(AsCodeBlock(ch.Stmt.body), cx.WithTarget(r), true).target; - } - if (r != null) - { - reachable = true; - if (cx.next.formalParameters.Length == 1) - r.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); - else { - Function overload = r.popOverload(); - r.WriteLine("loopDepth = {0};", cx.next.call(ch.Stmt.wait.result.name, "loopDepth")); - if (overload != null) { - overload.WriteLine("loopDepth = {0};", cx.next.call(string.Format("std::move({0})", ch.Stmt.wait.result.name), "loopDepth")); - r.setOverload(overload); - } - } - } - - var cbFunc = new Function { - name = "callback_fire", - returnType = "void", - formalParameters = new string[] { - ch.CallbackTypeInStateClass + "*", - ch.Stmt.wait.result.type + " const& value" - }, - endIsUnreachable = true - }; - cbFunc.addOverload(ch.CallbackTypeInStateClass + "*", ch.Stmt.wait.result.type + " && value"); - functions.Add(string.Format("{0}#{1}", cbFunc.name, ch.Index), cbFunc); - cbFunc.Indent(codeIndent); - ProbeEnter(cbFunc, actor.name, ch.Index); - cbFunc.WriteLine("{0};", exitFunc.call()); - - Function _overload = cbFunc.popOverload(); - TryCatch(cx.WithTarget(cbFunc), cx.catchFErr, cx.tryLoopDepth, () => { - cbFunc.WriteLine("{0};", ch.Body.call("value", "0")); - }, false); - if (_overload != null) { - TryCatch(cx.WithTarget(_overload), cx.catchFErr, cx.tryLoopDepth, () => { - _overload.WriteLine("{0};", ch.Body.call("std::move(value)", "0")); - }, false); - cbFunc.setOverload(_overload); - } - ProbeExit(cbFunc, actor.name, ch.Index); - - var errFunc = new Function - { - name = "callback_error", - returnType = "void", - formalParameters = new string[] { - ch.CallbackTypeInStateClass + "*", - "Error err" - }, - endIsUnreachable = true - }; - functions.Add(string.Format("{0}#{1}", errFunc.name, ch.Index), errFunc); - errFunc.Indent(codeIndent); - ProbeEnter(errFunc, actor.name, ch.Index); - errFunc.WriteLine("{0};", exitFunc.call()); - TryCatch(cx.WithTarget(errFunc), cx.catchFErr, cx.tryLoopDepth, () => - { - errFunc.WriteLine("{0};", cx.catchFErr.call("err", "0")); - }, false); - ProbeExit(errFunc, actor.name, ch.Index); - } - - bool firstChoice = true; - foreach (var ch in choices) - { - string getFunc = ch.Stmt.wait.isWaitNext ? "pop" : "get"; - LineNumber(cx.target, ch.Stmt.wait.FirstSourceLine); - if (ch.Stmt.wait.isWaitNext) { - cx.target.WriteLine("auto {0} = {1};", ch.Future, ch.Stmt.wait.futureExpression); - cx.target.WriteLine("static_assert(std::is_same>::value || std::is_same>::value, \"invalid type\");", ch.Future, ch.Stmt.wait.result.type); - - } else { - cx.target.WriteLine("{2}<{3}> {0} = {1};", ch.Future, ch.Stmt.wait.futureExpression, "StrictFuture", ch.Stmt.wait.result.type); - } - - if (firstChoice) - { - // Do this check only after evaluating the expression for the first wait expression, so that expression cannot be short circuited by cancellation. - // So wait( expr() ) will always evaluate `expr()`, but choose { when ( wait(success( expr2() )) {} } need - // not evaluate `expr2()`. - firstChoice = false; - LineNumber(cx.target, stmt.FirstSourceLine); - if (actor.IsCancellable()) - cx.target.WriteLine("if ({1}->actor_wait_state < 0) return {0};", cx.catchFErr.call("actor_cancelled()", AdjustLoopDepth(cx.tryLoopDepth)), This); - } - - cx.target.WriteLine("if ({0}.isReady()) {{ if ({0}.isError()) return {2}; else return {1}; }};", ch.Future, ch.Body.call(ch.Future + "." + getFunc + "()", "loopDepth"), cx.catchFErr.call(ch.Future + ".getError()", AdjustLoopDepth(cx.tryLoopDepth))); - } - cx.target.WriteLine("{1}->actor_wait_state = {0};", group, This); - foreach (var ch in choices) - { - LineNumber(cx.target, ch.Stmt.wait.FirstSourceLine); - cx.target.WriteLine("{0}.addCallbackAndClear(static_cast<{1}*>({2}));", ch.Future, ch.CallbackTypeInStateClass, This); - } - cx.target.WriteLine("loopDepth = 0;");//cx.target.WriteLine("return 0;"); - - if (!reachable) cx.unreachable(); - } - void CompileStatement(BreakStatement stmt, Context cx) - { - if (cx.breakF == null) - throw new Error(stmt.FirstSourceLine, "break outside loop"); - if (cx.breakF is LiteralBreak) - cx.target.WriteLine("{0};", cx.breakF.call()); - else - { - cx.target.WriteLine("return {0}; // break", cx.breakF.call("loopDepth==0?0:loopDepth-1")); - } - cx.unreachable(); - } - void CompileStatement(ContinueStatement stmt, Context cx) - { - if (cx.continueF == null) - throw new Error(stmt.FirstSourceLine, "continue outside loop"); - if (cx.continueF is LiteralContinue) - cx.target.WriteLine("{0};", cx.continueF.call()); - else - cx.target.WriteLine("return {0}; // continue", cx.continueF.call("loopDepth")); - cx.unreachable(); - } - void CompileStatement(WaitStatement stmt, Context cx) - { - var equiv = new ChooseStatement - { - body = new CodeBlock - { - statements = new Statement[] { - new WhenStatement { - wait = stmt, - body = null, - FirstSourceLine = stmt.FirstSourceLine, - } - }, - FirstSourceLine = stmt.FirstSourceLine - }, - FirstSourceLine = stmt.FirstSourceLine - }; - if (!stmt.resultIsState) { - cx.next.formalParameters = new string[] { - string.Format("{0} const& {1}", stmt.result.type, stmt.result.name), - loopDepth }; - cx.next.addOverload( - string.Format("{0} && {1}", stmt.result.type, stmt.result.name), - loopDepth); - } - CompileStatement(equiv, cx); - } - void CompileStatement(CodeBlock stmt, Context cx) - { - cx.target.WriteLine("{"); - cx.target.Indent(+1); - var end = Compile(stmt, cx, true); - cx.target.Indent(-1); - cx.target.WriteLine("}"); - if (end.target == null) - cx.unreachable(); - else if (end.target != cx.target) - end.target.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); - } - void CompileStatement(ReturnStatement stmt, Context cx) - { - LineNumber(cx.target, stmt.FirstSourceLine); - if ((stmt.expression == "") != (actor.returnType == null)) - throw new Error(stmt.FirstSourceLine, "Return statement does not match actor declaration"); - if (actor.returnType != null) - { - if (stmt.expression == "Never()") - { - // `return Never();` destroys state immediately but never returns to the caller - cx.target.WriteLine("this->~{0}();", stateClassName); - cx.target.WriteLine("{0}->sendAndDelPromiseRef(Never());", This); - } - else - { - // Short circuit if there are no futures outstanding, but still evaluate the expression - // if it has side effects - cx.target.WriteLine("if (!{0}->SAV<{1}>::futures) {{ (void)({2}); this->~{3}(); {0}->destroy(); return 0; }}", This, actor.returnType, stmt.expression, stateClassName); - // Build the return value directly in SAV::value_storage - // If the expression is exactly the name of a state variable, std::move() it - if (state.Exists(s => s.name == stmt.expression)) - { - cx.target.WriteLine("new (&{0}->SAV< {1} >::value()) {1}(std::move({2})); // state_var_RVO", This, actor.returnType, stmt.expression); - } - else - { - cx.target.WriteLine("new (&{0}->SAV< {1} >::value()) {1}({2});", This, actor.returnType, stmt.expression); - } - // Destruct state - cx.target.WriteLine("this->~{0}();", stateClassName); - // Tell SAV to return the value we already constructed in value_storage - cx.target.WriteLine("{0}->finishSendAndDelPromiseRef();", This); - } - } else - cx.target.WriteLine("delete {0};", This); - cx.target.WriteLine("return 0;"); - cx.unreachable(); - } - void CompileStatement(IfStatement stmt, Context cx) - { - bool useContinuation = WillContinue(stmt.ifBody) || WillContinue(stmt.elseBody); - - LineNumber(cx.target, stmt.FirstSourceLine); - cx.target.WriteLine("if {1}({0})", stmt.expression, stmt.constexpr ? "constexpr " : ""); - cx.target.WriteLine("{"); - cx.target.Indent(+1); - Function ifTarget = Compile(AsCodeBlock(stmt.ifBody), cx, useContinuation).target; - if (useContinuation && ifTarget != null) - ifTarget.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); - cx.target.Indent(-1); - cx.target.WriteLine("}"); - Function elseTarget = null; - if (stmt.elseBody != null || useContinuation) - { - cx.target.WriteLine("else"); - cx.target.WriteLine("{"); - cx.target.Indent(+1); - elseTarget = cx.target; - if (stmt.elseBody != null) - { - elseTarget = Compile(AsCodeBlock(stmt.elseBody), cx, useContinuation).target; - } - if (useContinuation && elseTarget != null) - elseTarget.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); - cx.target.Indent(-1); - cx.target.WriteLine("}"); - } - if (ifTarget == null && stmt.elseBody != null && elseTarget == null) - cx.unreachable(); - else if (!cx.next.wasCalled && useContinuation) - throw new Exception("Internal error: IfStatement: next not called?"); - } - void CompileStatement(TryStatement stmt, Context cx) - { - bool reachable = false; - - if (stmt.catches.Count != 1) throw new Error(stmt.FirstSourceLine, "try statement must have exactly one catch clause"); - var c = stmt.catches[0]; - string catchErrorParameterName = ""; - if (c.expression != "...") { - string exp = c.expression.Replace(" ",""); - if (!exp.StartsWith("Error&")) - throw new Error(c.FirstSourceLine, "Only type 'Error' or '...' may be caught in an actor function"); - catchErrorParameterName = exp.Substring(6); - } - if (catchErrorParameterName == "") catchErrorParameterName = "__current_error"; - - var catchFErr = getFunction(cx.target.name, "Catch", "const Error& " + catchErrorParameterName, loopDepth0); - catchFErr.exceptionParameterIs = catchErrorParameterName; - var end = TryCatchCompile(AsCodeBlock(stmt.tryBody), cx.WithCatch(catchFErr)); - if (end.target != null) reachable = true; - - if (end.target!=null) - TryCatch(end, cx.catchFErr, cx.tryLoopDepth, () => - end.target.WriteLine("loopDepth = {0};", cx.next.call("loopDepth"))); - - // Now to write the catch function - TryCatch(cx.WithTarget(catchFErr), cx.catchFErr, cx.tryLoopDepth, () => - { - var cend = Compile(AsCodeBlock(c.body), cx.WithTarget(catchFErr), true); - if (cend.target != null) cend.target.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); - if (cend.target != null) reachable = true; - }); - - if (!reachable) cx.unreachable(); - } - void CompileStatement(ThrowStatement stmt, Context cx) - { - LineNumber(cx.target, stmt.FirstSourceLine); - - if (stmt.expression == "") - { - if (cx.target.exceptionParameterIs != null) - cx.target.WriteLine("return {0};", cx.catchFErr.call(cx.target.exceptionParameterIs, AdjustLoopDepth( cx.tryLoopDepth ))); - else - throw new Error(stmt.FirstSourceLine, "throw statement with no expression has no current exception in scope"); - } - else - cx.target.WriteLine("return {0};", cx.catchFErr.call(stmt.expression, AdjustLoopDepth( cx.tryLoopDepth ))); - cx.unreachable(); - } - void CompileStatement(Statement stmt, Context cx) - { - // Use reflection for double dispatch. SOMEDAY: Use a Dictionary> and expression trees to memoize - var method = typeof(ActorCompiler).GetMethod("CompileStatement", - System.Reflection.BindingFlags.NonPublic|System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.ExactBinding, - null, new Type[] { stmt.GetType(), typeof(Context) }, null); - if (method == null) - throw new Error(stmt.FirstSourceLine, "Statement type {0} not supported yet.", stmt.GetType().Name); - try - { - method.Invoke(this, new object[] { stmt, cx }); - } - catch (System.Reflection.TargetInvocationException e) - { - if (!(e.InnerException is Error)) - Console.Error.WriteLine("\tHit error <{0}> for statement type {1} at line {2}\n\tStack Trace:\n{3}", - e.InnerException.Message, stmt.GetType().Name, stmt.FirstSourceLine, e.InnerException.StackTrace); - throw e.InnerException; - } - } - - // Compile returns a new context based on the one that is passed in, but (unlike CompileStatement) - // does not modify its parameter - // The target of the returned context is null if the end of the CodeBlock is unreachable (otherwise - // it is the target Function to which the end of the CodeBlock was written) - Context Compile(CodeBlock block, Context context, bool okToContinue=true) - { - var cx = context.Clone(); cx.next = null; - foreach (var stmt in block.statements) - { - if (cx.target == null) - { - throw new Error(stmt.FirstSourceLine, "Unreachable code."); - //Console.Error.WriteLine("\t(WARNING) Unreachable code at line {0}.", stmt.FirstSourceLine); - //break; - } - if (cx.next == null) - cx.next = getFunction(cx.target.name, "cont", loopDepth); - CompileStatement(stmt, cx); - if (cx.next.wasCalled) - { - if (cx.target == null) throw new Exception("Unreachable continuation called?"); - if (!okToContinue) throw new Exception("Unexpected continuation"); - cx.target = cx.next; - cx.next = null; - } - } - return cx; - } - - Dictionary functions = new Dictionary(); - - void WriteFunctions(TextWriter writer) - { - foreach (var func in functions.Values) - { - string body = func.BodyText; - if (body.Length != 0) - { - WriteFunction(writer, func, body); - } - if (func.overload != null) - { - string overloadBody = func.overload.BodyText; - if (overloadBody.Length != 0) - { - WriteFunction(writer, func.overload, overloadBody); - } - } - } - } - - private static void WriteFunction(TextWriter writer, Function func, string body) - { - writer.WriteLine(memberIndentStr + "{0}{1}({2}){3}", - func.returnType == "" ? "" : func.returnType + " ", - func.useByName(), - string.Join(",", func.formalParameters), - func.specifiers == "" ? "" : " " + func.specifiers); - if (func.returnType != "") - writer.WriteLine(memberIndentStr + "{"); - writer.WriteLine(body); - if (!func.endIsUnreachable) - writer.WriteLine(memberIndentStr + "\treturn loopDepth;"); - writer.WriteLine(memberIndentStr + "}"); - } - - Function getFunction(string baseName, string addName, string[] formalParameters, string[] overloadFormalParameters) - { - string proposedName; - if (addName == "cont" && baseName.Length>=5 && baseName.Substring(baseName.Length - 5, 4) == "cont") - proposedName = baseName.Substring(0, baseName.Length - 1); - else - proposedName = baseName + addName; - - int i = 0; - while (functions.ContainsKey(string.Format("{0}{1}", proposedName, ++i))) ; - - var f = new Function { - name = string.Format("{0}{1}", proposedName, i), - returnType = "int", - formalParameters = formalParameters - }; - if (overloadFormalParameters != null) { - f.addOverload(overloadFormalParameters); - } - f.Indent(codeIndent); - functions.Add(f.name, f); - return f; - } - - Function getFunction(string baseName, string addName, params string[] formalParameters) - { - return getFunction(baseName, addName, formalParameters, null); - } - - string[] ParameterList() - { - return actor.parameters.Select(p => - { - // SOMEDAY: pass small built in types by value - if (p.initializer != "") - return string.Format("{0} const& {1} = {2}", p.type, p.name, p.initializer); - else - return string.Format("{0} const& {1}", p.type, p.name); - }).ToArray(); - } - void WriteCancelFunc(TextWriter writer) - { - if (actor.IsCancellable()) - { - Function cancelFunc = new Function - { - name = "cancel", - returnType = "void", - formalParameters = new string[] {}, - endIsUnreachable = true, - publicName = true, - specifiers = "override" - }; - cancelFunc.Indent(codeIndent); - cancelFunc.WriteLine("auto wait_state = this->actor_wait_state;"); - cancelFunc.WriteLine("this->actor_wait_state = -1;"); - cancelFunc.WriteLine("switch (wait_state) {"); - int lastGroup = -1; - foreach (var cb in callbacks.OrderBy(cb => cb.CallbackGroup)) - if (cb.CallbackGroup != lastGroup) - { - lastGroup = cb.CallbackGroup; - cancelFunc.WriteLine("case {0}: this->a_callback_error(({1}*)0, actor_cancelled()); break;", cb.CallbackGroup, cb.type); - } - cancelFunc.WriteLine("}"); - WriteFunction(writer, cancelFunc, cancelFunc.BodyText); - } - } - - void WriteConstructor(Function body, TextWriter writer, string fullStateClassName) - { - Function constructor = new Function - { - name = className, - returnType = "", - formalParameters = ParameterList(), - endIsUnreachable = true, - publicName = true - }; - - // Initializes class member variables - constructor.Indent(codeIndent); - constructor.WriteLine( " : Actor<" + (actor.returnType == null ? "void" : actor.returnType) + ">()," ); - constructor.WriteLine( " {0}({1}),", fullStateClassName, string.Join(", ", actor.parameters.Select(p => p.name))); - constructor.WriteLine( " activeActorHelper(__actorIdentifier)"); - constructor.Indent(-1); - - constructor.WriteLine("{"); - constructor.Indent(+1); - - ProbeEnter(constructor, actor.name); - - constructor.WriteLine("#ifdef ENABLE_SAMPLING"); - constructor.WriteLine("this->lineage.setActorName(\"{0}\");", actor.name); - constructor.WriteLine("LineageScope _(&this->lineage);"); - // constructor.WriteLine("getCurrentLineage()->modify(&StackLineage::actorName) = \"{0}\"_sr;", actor.name); - constructor.WriteLine("#endif"); - - constructor.WriteLine("this->{0};", body.call()); - - ProbeExit(constructor, actor.name); - - WriteFunction(writer, constructor, constructor.BodyText); - } - - void WriteStateConstructor(TextWriter writer) - { - Function constructor = new Function - { - name = stateClassName, - returnType = "", - formalParameters = ParameterList(), - endIsUnreachable = true, - publicName = true - }; - constructor.Indent(codeIndent); - string ini = null; - int line = actor.SourceLine; - var initializers = state.AsEnumerable(); - foreach (var s in initializers) - if (s.initializer != null) - { - LineNumber(constructor, line); - if (ini != null) - { - constructor.WriteLine(ini + ","); - ini = " "; - } - else - { - ini = " : "; - } - - ini += string.Format("{0}({1})", s.name, s.initializer); - line = s.SourceLine; - } - LineNumber(constructor, line); - if (ini != null) - constructor.WriteLine(ini); - constructor.Indent(-1); - constructor.WriteLine("{"); - constructor.Indent(+1); - ProbeCreate(constructor, actor.name); - WriteFunction(writer, constructor, constructor.BodyText); - } - - void WriteStateDestructor(TextWriter writer) { - Function destructor = new Function - { - name = String.Format("~{0}", stateClassName), - returnType = "", - formalParameters = new string[0], - endIsUnreachable = true, - publicName = true, - }; - destructor.Indent(codeIndent); - destructor.Indent(-1); - destructor.WriteLine("{"); - destructor.Indent(+1); - ProbeDestroy(destructor, actor.name); - WriteFunction(writer, destructor, destructor.BodyText); - } - - IEnumerable Flatten(Statement stmt) - { - if (stmt == null) return new Statement[] { }; - var fl = new TypeSwitch>(stmt) - .Case(s => Flatten(s.body)) - .Case(s => Flatten(s.body)) - .Case(s => Flatten(s.body)) - .Case(s => Flatten(s.body)) - .Case(s => s.statements.SelectMany(t=>Flatten(t))) - .Case( s => Flatten(s.ifBody).Concat(Flatten(s.elseBody)) ) - .Case( s => Flatten(s.body) ) - .Case( s => Flatten(s.body) ) - .Case( s => Flatten(s.tryBody).Concat( s.catches.SelectMany(c=>Flatten(c.body)) ) ) - .Case(s => Enumerable.Empty()) - .Return(); - return new Statement[]{stmt}.Concat(fl); - } - - void FindState() - { - state = actor.parameters - .Select( - p=>new StateVar { SourceLine = actor.SourceLine, name=p.name, type=p.type, initializer=p.name, initializerConstructorSyntax=false } ) - .ToList(); - } - - // Generate an expression equivalent to max(0, loopDepth-subtract) for the given constant subtract - string AdjustLoopDepth(int subtract) - { - if (subtract == 0) - return "loopDepth"; - else - return string.Format("std::max(0, loopDepth - {0})", subtract); - } - } -} diff --git a/flow/actorcompiler/ActorCompiler.targets b/flow/actorcompiler/ActorCompiler.targets deleted file mode 100644 index 16eaec6a8af..00000000000 --- a/flow/actorcompiler/ActorCompiler.targets +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - true - - - - - - - - _ActorCompiler - - - - - - - - - - $(BuildGenerateSourcesTargets); - ComputeACOutput; - - - - - - - - - - - - - - - - diff --git a/flow/actorcompiler/ActorCompiler.xml b/flow/actorcompiler/ActorCompiler.xml deleted file mode 100644 index 19ed212ddcf..00000000000 --- a/flow/actorcompiler/ActorCompiler.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - Options - - - - - Command Line - - - - - - Compile generated file - - - The resulting file is a C++ module to be compiled, not a header file to be included - - - - - Actor Compiler Options - - - Actor Compiler Options - - - - - Additional C++ Options - - - Options passed to the C++ compiler processing the generated code - - - - - - - - - - \ No newline at end of file diff --git a/flow/actorcompiler/ActorParser.cs b/flow/actorcompiler/ActorParser.cs deleted file mode 100644 index 7eb5cf53cb0..00000000000 --- a/flow/actorcompiler/ActorParser.cs +++ /dev/null @@ -1,1178 +0,0 @@ -/* - * ActorParser.cs - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; - -namespace actorcompiler -{ - class Error : Exception - { - public int SourceLine { get; private set; } - public Error(int SourceLine, string format, params object[] args) - : base(string.Format(format,args)) - { - this.SourceLine = SourceLine; - } - }; - - class ErrorMessagePolicy - { - public bool DisableDiagnostics = false; - public void HandleActorWithoutWait(String sourceFile, Actor actor) - { - if (!DisableDiagnostics && !actor.isTestCase) - { - // TODO(atn34): Once cmake is the only build system we can make this an error instead of a warning. - Console.Error.WriteLine("{0}:{1}: warning: ACTOR {2} does not contain a wait() statement", sourceFile, actor.SourceLine, actor.name); - } - } - public bool ActorsNoDiscardByDefault() { - return !DisableDiagnostics; - } - } - - class Token - { - public string Value; - public int Position; - public int SourceLine; - public int BraceDepth; - public int ParenDepth; - public bool IsWhitespace { get { return Value == " " || Value == "\n" || Value == "\r" || Value == "\r\n" || Value == "\t" || Value.StartsWith("//") || Value.StartsWith("/*"); } } - public override string ToString() { return Value; } - public Token Assert(string error, Func pred) - { - if (!pred(this)) throw new Error(SourceLine, error); - return this; - } - public TokenRange GetMatchingRangeIn(TokenRange range) - { - Func pred; - int dir; - switch (Value) { - case "(": pred = t=> t.Value != ")" || t.ParenDepth != ParenDepth; dir = +1; break; - case ")": pred = t=> t.Value != "(" || t.ParenDepth != ParenDepth; dir = -1; break; - case "{": pred = t=> t.Value != "}" || t.BraceDepth != BraceDepth; dir = +1; break; - case "}": pred = t=> t.Value != "{" || t.BraceDepth != BraceDepth; dir = -1; break; - case "<": return - new TokenRange(range.GetAllTokens(), - Position+1, - AngleBracketParser.NotInsideAngleBrackets( - new TokenRange(range.GetAllTokens(), Position, range.End)) - .Skip(1) // skip the "<", which is considered "outside" - .First() // get the ">", which is likewise "outside" - .Position); - case "[": return - new TokenRange(range.GetAllTokens(), - Position+1, - BracketParser.NotInsideBrackets( - new TokenRange(range.GetAllTokens(), Position, range.End)) - .Skip(1) // skip the "[", which is considered "outside" - .First() // get the "]", which is likewise "outside" - .Position); - default: throw new NotSupportedException("Can't match this token!"); - } - TokenRange r; - if (dir == -1) - { - r = new TokenRange(range.GetAllTokens(), range.Begin, Position) - .RevTakeWhile(pred); - if (r.Begin == range.Begin) - throw new Error(SourceLine, "Syntax error: Unmatched " + Value); - } - else - { - r = new TokenRange(range.GetAllTokens(), Position+1, range.End) - .TakeWhile(pred); - if (r.End == range.End) - throw new Error(SourceLine, "Syntax error: Unmatched " + Value); - } - return r; - } - }; - - class TokenRange : IEnumerable - { - public TokenRange(Token[] tokens, int beginPos, int endPos) - { - if (beginPos > endPos) throw new InvalidOperationException("Invalid TokenRange"); - this.tokens = tokens; - this.beginPos = beginPos; - this.endPos = endPos; - } - - public bool IsEmpty { get { return beginPos==endPos; } } - public int Begin { get { return beginPos; } } - public int End { get { return endPos; } } - public Token First() { - if (beginPos == endPos) throw new InvalidOperationException("Empty TokenRange"); - return tokens[beginPos]; - } - public Token Last() { - if (beginPos == endPos) throw new InvalidOperationException("Empty TokenRange"); - return tokens[endPos - 1]; - } - public Token Last(Func pred) - { - for (int i = endPos - 1; i >= beginPos; i--) - if (pred(tokens[i])) - return tokens[i]; - throw new Exception("Matching token not found"); - } - public TokenRange Skip(int count) - { - return new TokenRange(tokens, beginPos + count, endPos); - } - public TokenRange Consume(string value) - { - First().Assert("Expected " + value, t => t.Value == value); - return Skip(1); - } - public TokenRange Consume(string error, Func pred) - { - First().Assert(error, pred); - return Skip(1); - } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return GetEnumerator(); } - public IEnumerator GetEnumerator() - { - for (int i = beginPos; i < endPos; i++) - yield return tokens[i]; - } - public TokenRange SkipWhile(Func pred) - { - for (int e = beginPos; e < endPos; e++) - if (!pred(tokens[e])) - return new TokenRange(tokens, e, endPos); - return new TokenRange(tokens, endPos, endPos); - } - public TokenRange TakeWhile(Func pred) - { - for (int e = beginPos; e < endPos; e++) - if (!pred(tokens[e])) - return new TokenRange(tokens, beginPos, e); - return new TokenRange(tokens, beginPos, endPos); - } - public TokenRange RevTakeWhile(Func pred) - { - for (int e = endPos-1; e >= beginPos; e--) - if (!pred(tokens[e])) - return new TokenRange(tokens, e+1, endPos); - return new TokenRange(tokens, beginPos, endPos); - } - public TokenRange RevSkipWhile(Func pred) - { - for (int e = endPos - 1; e >= beginPos; e--) - if (!pred(tokens[e])) - return new TokenRange(tokens, beginPos, e + 1); - return new TokenRange(tokens, beginPos, beginPos); - } - public Token[] GetAllTokens() { return tokens; } - - public int Length { - get { - return endPos - beginPos; - } - } - - Token[] tokens; - int beginPos; - int endPos; - }; - - static class BracketParser - { - public static IEnumerable NotInsideBrackets(IEnumerable tokens) - { - int BracketDepth = 0; - int? BasePD = null; - foreach (var tok in tokens) - { - if (BasePD == null) BasePD = tok.ParenDepth; - if (tok.ParenDepth == BasePD && tok.Value == "]") BracketDepth--; - if (BracketDepth == 0) - yield return tok; - if (tok.ParenDepth == BasePD && tok.Value == "[") BracketDepth++; - } - } - }; - static class AngleBracketParser - { - public static IEnumerable NotInsideAngleBrackets(IEnumerable tokens) - { - int AngleDepth = 0; - int? BasePD = null; - foreach (var tok in tokens) - { - if (BasePD == null) BasePD = tok.ParenDepth; - if (tok.ParenDepth == BasePD && tok.Value == ">") AngleDepth--; - if (AngleDepth == 0) - yield return tok; - if (tok.ParenDepth == BasePD && tok.Value == "<") AngleDepth++; - } - } - }; - - class ActorParser - { - public bool LineNumbersEnabled = true; - - Token[] tokens; - string sourceFile; - ErrorMessagePolicy errorMessagePolicy; - public bool generateProbes; - public Dictionary<(ulong, ulong), string> uidObjects { get; private set; } - - public ActorParser(string text, string sourceFile, ErrorMessagePolicy errorMessagePolicy, bool generateProbes) - { - this.sourceFile = sourceFile; - this.errorMessagePolicy = errorMessagePolicy; - this.generateProbes = generateProbes; - this.uidObjects = new Dictionary<(ulong, ulong), string>(); - tokens = Tokenize(text).Select(t=>new Token{ Value=t }).ToArray(); - CountParens(); - //if (sourceFile.EndsWith(".h")) LineNumbersEnabled = false; - //Console.WriteLine("{0} chars -> {1} tokens", text.Length, tokens.Length); - //showTokens(); - } - - class ClassContext { - public string name; - public int inBlocks; - } - - private bool ParseClassContext(TokenRange toks, out string name) - { - name = ""; - if (toks.Begin == toks.End) - { - return false; - } - - // http://nongnu.org/hcb/#attribute-specifier-seq - Token first; - while (true) - { - first = toks.First(NonWhitespace); - if (first.Value == "[") - { - var contents = first.GetMatchingRangeIn(toks); - toks = range(contents.End + 1, toks.End); - } - else if (first.Value == "alignas") - { - toks = range(first.Position + 1, toks.End); - first = toks.First(NonWhitespace); - first.Assert("Expected ( after alignas", t => t.Value == "("); - var contents = first.GetMatchingRangeIn(toks); - toks = range(contents.End + 1, toks.End); - } - else - { - break; - } - } - - // http://nongnu.org/hcb/#class-head-name - first = toks.First(NonWhitespace); - if (!identifierPattern.Match(first.Value).Success) { - return false; - } - while (true) { - first.Assert("Expected identifier", t=>identifierPattern.Match(t.Value).Success); - name += first.Value; - toks = range(first.Position + 1, toks.End); - if (toks.First(NonWhitespace).Value == "::") { - name += "::"; - toks = toks.SkipWhile(Whitespace).Skip(1); - } else { - break; - } - first = toks.First(NonWhitespace); - } - // http://nongnu.org/hcb/#class-virt-specifier-seq - toks = toks.SkipWhile(t => Whitespace(t) || t.Value == "final" || t.Value == "explicit"); - - first = toks.First(NonWhitespace); - if (first.Value == ":" || first.Value == "{") { - // At this point we've confirmed that this is a class. - return true; - } - return false; - } - - public void Write(System.IO.TextWriter writer, string destFileName) - { - writer.NewLine = "\n"; - writer.WriteLine("#define POST_ACTOR_COMPILER 1"); - int outLine = 1; - if (LineNumbersEnabled) - { - writer.WriteLine("#line {0} \"{1}\"", tokens[0].SourceLine, sourceFile); - outLine++; - } - int inBlocks = 0; - Stack classContextStack = new Stack(); - for(int i=0; i 0) - { - actor.enclosingClass = String.Join("::", classContextStack.Reverse().Select(t => t.name)); - } - var actorWriter = new System.IO.StringWriter(); - actorWriter.NewLine = "\n"; - var actorCompiler = new ActorCompiler(actor, sourceFile, inBlocks == 0, LineNumbersEnabled, generateProbes); - actorCompiler.Write(actorWriter); - actorCompiler.uidObjects.ToList().ForEach(x => this.uidObjects.TryAdd(x.Key, x.Value)); - - string[] actorLines = actorWriter.ToString().Split('\n'); - - bool hasLineNumber = false; - bool hadLineNumber = true; - foreach (var line in actorLines) - { - if (LineNumbersEnabled) - { - bool isLineNumber = line.Contains("#line"); - if (isLineNumber) hadLineNumber = true; - if (!isLineNumber && !hasLineNumber && hadLineNumber) - { - writer.WriteLine("\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} \"{1}\"", outLine + 1, destFileName); - outLine++; - hadLineNumber = false; - } - hasLineNumber = isLineNumber; - } - writer.WriteLine(line.TrimEnd('\n','\r')); - outLine++; - } - - i = end; - if (i != tokens.Length && LineNumbersEnabled) - { - writer.WriteLine("#line {0} \"{1}\"", tokens[i].SourceLine, sourceFile); - outLine++; - } - } - else if (tokens[i].Value == "DESCR") - { - int end; - var descr = ParseDescr(i, out end); - int lines; - new DescrCompiler(descr, tokens[i].BraceDepth).Write(writer, out lines); - i = end; - outLine += lines; - if (i != tokens.Length && LineNumbersEnabled) - { - writer.WriteLine("#line {0} \"{1}\"", tokens[i].SourceLine, sourceFile); - outLine++; - } - } - else if (tokens[i].Value == "class" || tokens[i].Value == "struct" || tokens[i].Value == "union") - { - writer.Write(tokens[i].Value); - string name; - if (ParseClassContext(range(i+1, tokens.Length), out name)) - { - classContextStack.Push(new ClassContext { name = name, inBlocks = inBlocks}); - } - } - else - { - if (tokens[i].Value == "{") - { - inBlocks++; - } - else if (tokens[i].Value == "}") - { - inBlocks--; - if (classContextStack.Count > 0 && classContextStack.Peek().inBlocks == inBlocks) - { - classContextStack.Pop(); - } - } - writer.Write(tokens[i].Value); - outLine += tokens[i].Value.Count(c => c == '\n'); - } - } - } - - IEnumerable SplitParameterList( TokenRange toks, string delimiter ) { - if (toks.Begin==toks.End) yield break; - while (true) { - Token comma = AngleBracketParser.NotInsideAngleBrackets( toks ) - .FirstOrDefault( t=> t.Value==delimiter && t.ParenDepth == toks.First().ParenDepth ); - if (comma == null) break; - yield return range(toks.Begin,comma.Position); - toks = range(comma.Position + 1, toks.End); - } - yield return toks; - } - - IEnumerable NormalizeWhitespace(IEnumerable tokens) - { - bool inWhitespace = false; - bool leading = true; - foreach (var tok in tokens) - { - if (!tok.IsWhitespace) - { - if (inWhitespace && !leading) yield return new Token { Value = " " }; - inWhitespace = false; - yield return tok; - leading = false; - } - else - { - inWhitespace = true; - } - } - } - - void ParseDeclaration(TokenRange tokens, - out Token name, - out TokenRange type, - out TokenRange initializer, - out bool constructorSyntax) - { - initializer = null; - TokenRange beforeInitializer = tokens; - constructorSyntax = false; - - Token equals = AngleBracketParser.NotInsideAngleBrackets(tokens) - .FirstOrDefault(t => t.Value == "=" && t.ParenDepth == tokens.First().ParenDepth); - if (equals != null) - { - // type name = initializer; - beforeInitializer = range(tokens.Begin,equals.Position); - initializer = range(equals.Position + 1, tokens.End); - } - else - { - Token paren = AngleBracketParser.NotInsideAngleBrackets(tokens) - .FirstOrDefault(t => t.Value == "("); - if (paren != null) - { - // type name(initializer); - constructorSyntax = true; - beforeInitializer = range(tokens.Begin, paren.Position); - initializer = - range(paren.Position + 1, tokens.End) - .TakeWhile(t => t.ParenDepth > paren.ParenDepth); - } else { - Token brace = AngleBracketParser.NotInsideAngleBrackets(tokens).FirstOrDefault(t => t.Value == "{"); - if (brace != null) { - // type name{initializer}; - throw new Error(brace.SourceLine, "Uniform initialization syntax is not currently supported for state variables (use '(' instead of '}}' ?)"); - } - } - } - name = beforeInitializer.Last(NonWhitespace); - if (beforeInitializer.Begin == name.Position) - throw new Error(beforeInitializer.First().SourceLine, "Declaration has no type."); - type = range(beforeInitializer.Begin, name.Position); - } - - VarDeclaration ParseVarDeclaration(TokenRange tokens) - { - Token name; - TokenRange type, initializer; - bool constructorSyntax; - ParseDeclaration( tokens, out name, out type, out initializer, out constructorSyntax ); - return new VarDeclaration - { - name = name.Value, - type = str(NormalizeWhitespace(type)), - initializer = initializer == null ? "" : str(NormalizeWhitespace(initializer)), - initializerConstructorSyntax = constructorSyntax - }; - } - - readonly Func Whitespace = (Token t) => t.IsWhitespace; - readonly Func NonWhitespace = (Token t) => !t.IsWhitespace; - - void ParseDescrHeading(Descr descr, TokenRange toks) - { - toks.First(NonWhitespace).Assert("non-struct DESCR!", t => t.Value == "struct"); - toks = toks.SkipWhile(Whitespace).Skip(1).SkipWhile(Whitespace); - - var colon = toks.FirstOrDefault(t => t.Value == ":"); - if (colon != null) - { - descr.superClassList = str(range(colon.Position + 1, toks.End)).Trim(); - toks = range(toks.Begin, colon.Position); - } - descr.name = str(toks).Trim(); - } - - void ParseTestCaseHeading(Actor actor, TokenRange toks) - { - actor.isStatic = true; - - // The parameter(s) to the TEST_CASE macro are opaque to the actor compiler - TokenRange paramRange = toks.Last(NonWhitespace) - .Assert("Unexpected tokens after test case parameter list.", - t => t.Value == ")" && t.ParenDepth == toks.First().ParenDepth) - .GetMatchingRangeIn(toks); - actor.testCaseParameters = str(paramRange); - - actor.name = "flowTestCase" + toks.First().SourceLine; - actor.parameters = new VarDeclaration[] { new VarDeclaration { - name = "params", - type = "UnitTestParameters", - initializer = "", - initializerConstructorSyntax = false - } - }; - actor.returnType = "Void"; - } - - void ParseActorHeading(Actor actor, TokenRange toks) - { - var template = toks.First(NonWhitespace); - if (template.Value == "template") - { - var templateParams = range(template.Position+1, toks.End) - .First(NonWhitespace) - .Assert("Invalid template declaration", t=>t.Value=="<") - .GetMatchingRangeIn(toks); - - actor.templateFormals = SplitParameterList(templateParams, ",") - .Select(p => ParseVarDeclaration(p)) //< SOMEDAY: ? - .ToArray(); - - toks = range(templateParams.End + 1, toks.End); - } - var attribute = toks.First(NonWhitespace); - while (attribute.Value == "[") - { - var attributeContents = attribute.GetMatchingRangeIn(toks); - - var asArray = attributeContents.ToArray(); - if (asArray.Length < 2 || asArray[0].Value != "[" || asArray[asArray.Length - 1].Value != "]") - { - throw new Error(actor.SourceLine, "Invalid attribute: Expected [[...]]"); - } - actor.attributes.Add("[" + str(NormalizeWhitespace(attributeContents)) + "]"); - toks = range(attributeContents.End + 1, toks.End); - - attribute = toks.First(NonWhitespace); - } - - var staticKeyword = toks.First(NonWhitespace); - if (staticKeyword.Value == "static") - { - actor.isStatic = true; - toks = range(staticKeyword.Position + 1, toks.End); - } - - var uncancellableKeyword = toks.First(NonWhitespace); - if (uncancellableKeyword.Value == "UNCANCELLABLE") - { - actor.SetUncancellable(); - toks = range(uncancellableKeyword.Position + 1, toks.End); - } - - // Find the parameter list - TokenRange paramRange = toks.Last(NonWhitespace) - .Assert("Unexpected tokens after actor parameter list.", - t => t.Value == ")" && t.ParenDepth == toks.First().ParenDepth) - .GetMatchingRangeIn(toks); - actor.parameters = SplitParameterList(paramRange, ",") - .Select(p => ParseVarDeclaration(p)) - .ToArray(); - - var name = range(toks.Begin,paramRange.Begin-1).Last(NonWhitespace); - actor.name = name.Value; - - // SOMEDAY: refactor? - var returnType = range(toks.First().Position + 1, name.Position).SkipWhile(Whitespace); - var retToken = returnType.First(); - if (retToken.Value == "Future") - { - var ofType = returnType.Skip(1).First(NonWhitespace).Assert("Expected <", tok => tok.Value == "<").GetMatchingRangeIn(returnType); - actor.returnType = str(NormalizeWhitespace(ofType)); - toks = range(ofType.End + 1, returnType.End); - } - else if (retToken.Value == "void"/* && !returnType.Skip(1).Any(NonWhitespace)*/) - { - actor.returnType = null; - toks = returnType.Skip(1); - } - else - throw new Error(actor.SourceLine, "Actor apparently does not return Future"); - - toks = toks.SkipWhile(Whitespace); - if (!toks.IsEmpty) - { - if (toks.Last().Value == "::") - { - actor.nameSpace = str(range(toks.Begin, toks.End - 1)); - } - else - { - Console.WriteLine("Tokens: '{0}' {1} '{2}'", str(toks), toks.Count(), toks.Last().Value); - throw new Error(actor.SourceLine, "Unrecognized tokens preceding parameter list in actor declaration"); - } - } - if (errorMessagePolicy.ActorsNoDiscardByDefault() && !actor.attributes.Contains("[[flow_allow_discard]]")) { - if (actor.IsCancellable()) - { - actor.attributes.Add("[[nodiscard]]"); - } - } - HashSet knownFlowAttributes = new HashSet(); - knownFlowAttributes.Add("[[flow_allow_discard]]"); - foreach (var flowAttribute in actor.attributes.Where(a => a.StartsWith("[[flow_"))) { - if (!knownFlowAttributes.Contains(flowAttribute)) { - throw new Error(actor.SourceLine, "Unknown flow attribute {0}", flowAttribute); - } - } - actor.attributes = actor.attributes.Where(a => !a.StartsWith("[[flow_")).ToList(); - } - - LoopStatement ParseLoopStatement(TokenRange toks) - { - return new LoopStatement { - body = ParseCompoundStatement( toks.Consume("loop") ) - }; - } - - ChooseStatement ParseChooseStatement(TokenRange toks) - { - return new ChooseStatement - { - body = ParseCompoundStatement(toks.Consume("choose")) - }; - } - - WhenStatement ParseWhenStatement(TokenRange toks) - { - var expr = toks.Consume("when") - .SkipWhile(Whitespace) - .First() - .Assert("Expected (", t => t.Value == "(") - .GetMatchingRangeIn(toks) - .SkipWhile(Whitespace); - - return new WhenStatement { - wait = ParseWaitStatement(expr), - body = ParseCompoundStatement(range(expr.End+1, toks.End)) - }; - } - - StateDeclarationStatement ParseStateDeclaration(TokenRange toks) - { - toks = toks.Consume("state").RevSkipWhile(t => t.Value == ";"); - return new StateDeclarationStatement { - decl = ParseVarDeclaration(toks) - }; - } - - ReturnStatement ParseReturnStatement(TokenRange toks) - { - toks = toks.Consume("return").RevSkipWhile(t => t.Value == ";"); - return new ReturnStatement - { - expression = str(NormalizeWhitespace(toks)) - }; - } - - ThrowStatement ParseThrowStatement(TokenRange toks) - { - toks = toks.Consume("throw").RevSkipWhile(t => t.Value == ";"); - return new ThrowStatement - { - expression = str(NormalizeWhitespace(toks)) - }; - } - - WaitStatement ParseWaitStatement(TokenRange toks) - { - WaitStatement ws = new WaitStatement(); - ws.FirstSourceLine = toks.First().SourceLine; - if (toks.First().Value == "state") - { - ws.resultIsState = true; - toks = toks.Consume("state"); - } - TokenRange initializer; - if (toks.First().Value == "wait" || toks.First().Value == "waitNext") - { - initializer = toks.RevSkipWhile(t=>t.Value==";"); - ws.result = new VarDeclaration { - name = "_", - type = "Void", - initializer = "", - initializerConstructorSyntax = false - }; - } else { - Token name; - TokenRange type; - bool constructorSyntax; - ParseDeclaration( toks.RevSkipWhile(t=>t.Value==";"), out name, out type, out initializer, out constructorSyntax ); - - string typestring = str(NormalizeWhitespace(type)); - if (typestring == "Void") { - throw new Error(ws.FirstSourceLine, "Assigning the result of a Void wait is not allowed. Just use a standalone wait statement."); - } - - ws.result = new VarDeclaration - { - name = name.Value, - type = str(NormalizeWhitespace(type)), - initializer = "", - initializerConstructorSyntax = false - }; - } - - if (initializer == null) throw new Error(ws.FirstSourceLine, "Wait statement must be a declaration or standalone statement"); - - var waitParams = initializer - .SkipWhile(Whitespace).Consume("Statement contains a wait, but is not a valid wait statement or a supported compound statement.1", - t=> { - if (t.Value=="wait") return true; - if (t.Value=="waitNext") { ws.isWaitNext = true; return true; } - return false; - }) - .SkipWhile(Whitespace).First().Assert("Expected (", t => t.Value == "(") - .GetMatchingRangeIn(initializer); - if (!range(waitParams.End, initializer.End).Consume(")").All(Whitespace)) { - throw new Error(toks.First().SourceLine, "Statement contains a wait, but is not a valid wait statement or a supported compound statement.2"); - } - - ws.futureExpression = str(NormalizeWhitespace(waitParams)); - return ws; - } - - WhileStatement ParseWhileStatement(TokenRange toks) - { - var expr = toks.Consume("while") - .First(NonWhitespace) - .Assert("Expected (", t => t.Value == "(") - .GetMatchingRangeIn(toks); - return new WhileStatement - { - expression = str(NormalizeWhitespace(expr)), - body = ParseCompoundStatement(range(expr.End + 1, toks.End)) - }; - } - - Statement ParseForStatement(TokenRange toks) - { - var head = - toks.Consume("for") - .First(NonWhitespace) - .Assert("Expected (", t => t.Value == "(") - .GetMatchingRangeIn(toks); - - Token[] delim = - head.Where( - t => t.ParenDepth == head.First().ParenDepth && - t.BraceDepth == head.First().BraceDepth && - t.Value==";" - ).ToArray(); - if (delim.Length == 2) - { - var init = range(head.Begin, delim[0].Position); - var cond = range(delim[0].Position + 1, delim[1].Position); - var next = range(delim[1].Position + 1, head.End); - var body = range(head.End + 1, toks.End); - - return new ForStatement - { - initExpression = str(NormalizeWhitespace(init)), - condExpression = str(NormalizeWhitespace(cond)), - nextExpression = str(NormalizeWhitespace(next)), - body = ParseCompoundStatement(body) - }; - } - - delim = - head.Where( - t => t.ParenDepth == head.First().ParenDepth && - t.BraceDepth == head.First().BraceDepth && - t.Value == ":" - ).ToArray(); - if (delim.Length != 1) - { - throw new Error(head.First().SourceLine, "for statement must be 3-arg style or c++11 2-arg style"); - } - - return new RangeForStatement - { - // The container over which to iterate - rangeExpression = str(NormalizeWhitespace(range(delim[0].Position + 1, head.End).SkipWhile(Whitespace))), - // Type and name of the variable assigned in each iteration - rangeDecl = str(NormalizeWhitespace(range(head.Begin, delim[0].Position - 1).SkipWhile(Whitespace))), - // The body of the for loop - body = ParseCompoundStatement(range(head.End + 1, toks.End)) - }; - } - - Statement ParseIfStatement(TokenRange toks) - { - toks = toks.Consume("if"); - toks = toks.SkipWhile(Whitespace); - bool constexpr = toks.First().Value == "constexpr"; - if(constexpr) { - toks = toks.Consume("constexpr").SkipWhile(Whitespace); - } - - var expr = toks.First(NonWhitespace) - .Assert("Expected (", t => t.Value == "(") - .GetMatchingRangeIn(toks); - return new IfStatement { - expression = str(NormalizeWhitespace(expr)), - constexpr = constexpr, - ifBody = ParseCompoundStatement(range(expr.End+1, toks.End)) - // elseBody will be filled in later if necessary by ParseElseStatement - }; - } - void ParseElseStatement(TokenRange toks, Statement prevStatement) - { - var ifStatement = prevStatement as IfStatement; - while (ifStatement != null && ifStatement.elseBody != null) - ifStatement = ifStatement.elseBody as IfStatement; - if (ifStatement == null) - throw new Error(toks.First().SourceLine, "else without matching if"); - ifStatement.elseBody = ParseCompoundStatement(toks.Consume("else")); - } - - Statement ParseTryStatement(TokenRange toks) - { - return new TryStatement - { - tryBody = ParseCompoundStatement(toks.Consume("try")), - catches = new List() // will be filled in later by ParseCatchStatement - }; - } - void ParseCatchStatement(TokenRange toks, Statement prevStatement) - { - var tryStatement = prevStatement as TryStatement; - if (tryStatement == null) - throw new Error(toks.First().SourceLine, "catch without matching try"); - var expr = toks.Consume("catch") - .First(NonWhitespace) - .Assert("Expected (", t => t.Value == "(") - .GetMatchingRangeIn(toks); - tryStatement.catches.Add( - new TryStatement.Catch - { - expression = str(NormalizeWhitespace(expr)), - body = ParseCompoundStatement(range(expr.End + 1, toks.End)), - FirstSourceLine = expr.First().SourceLine - }); - } - - static readonly HashSet IllegalKeywords = new HashSet { "goto", "do", "finally", "__if_exists", "__if_not_exists" }; - - void ParseDeclaration(TokenRange toks, List declarations) - { - Declaration dec = new Declaration(); - - Token delim = toks.First(t => t.Value == ";"); - var nameRange = range(toks.Begin, delim.Position).RevSkipWhile(Whitespace).RevTakeWhile(NonWhitespace); - var typeRange = range(toks.Begin, nameRange.Begin); - var commentRange = range(delim.Position + 1, toks.End); - - dec.name = str(nameRange).Trim(); - dec.type = str(typeRange).Trim(); - dec.comment = str(commentRange).Trim().TrimStart('/'); - - declarations.Add(dec); - } - - void ParseStatement(TokenRange toks, List statements) - { - toks = toks.SkipWhile(Whitespace); - - Action Add = stmt => - { - stmt.FirstSourceLine = toks.First().SourceLine; - statements.Add(stmt); - }; - - switch (toks.First().Value) - { - case "loop": Add(ParseLoopStatement(toks)); break; - case "while": Add(ParseWhileStatement(toks)); break; - case "for": Add(ParseForStatement(toks)); break; - case "break": Add(new BreakStatement()); break; - case "continue": Add(new ContinueStatement()); break; - case "return": Add(ParseReturnStatement(toks)); break; - case "{": Add(ParseCompoundStatement(toks)); break; - case "if": Add(ParseIfStatement(toks)); break; - case "else": ParseElseStatement(toks, statements[statements.Count - 1]); break; - case "choose": Add(ParseChooseStatement(toks)); break; - case "when": Add(ParseWhenStatement(toks)); break; - case "try": Add(ParseTryStatement(toks)); break; - case "catch": ParseCatchStatement(toks, statements[statements.Count - 1]); break; - case "throw": Add(ParseThrowStatement(toks)); break; - default: - if (IllegalKeywords.Contains(toks.First().Value)) - throw new Error(toks.First().SourceLine, "Statement '{0}' not supported in actors.", toks.First().Value); - if (toks.Any(t => t.Value == "wait" || t.Value == "waitNext")) - Add(ParseWaitStatement(toks)); - else if (toks.First().Value == "state") - Add(ParseStateDeclaration(toks)); - else if (toks.First().Value == "switch" && toks.Any(t => t.Value == "return")) - throw new Error(toks.First().SourceLine, "Unsupported compound statement containing return."); - else if (toks.First().Value.StartsWith("#")) - throw new Error(toks.First().SourceLine, "Found \"{0}\". Preprocessor directives are not supported within ACTORs", toks.First().Value); - else if (toks.RevSkipWhile(t => t.Value == ";").Any(NonWhitespace)) - Add(new PlainOldCodeStatement - { - code = str(NormalizeWhitespace(toks.RevSkipWhile(t => t.Value == ";"))) + ";" - }); - break; - }; - } - - Statement ParseCompoundStatement(TokenRange toks) - { - var first = toks.First(NonWhitespace); - if (first.Value == "{") { - var inBraces = first.GetMatchingRangeIn(toks); - if (!range(inBraces.End, toks.End).Consume("}").All(Whitespace)) - throw new Error(inBraces.Last().SourceLine, "Unexpected tokens after compound statement"); - return ParseCodeBlock(inBraces); - } else { - List statements = new List(); - ParseStatement( toks.Skip(1), statements ); - return statements[0]; - } - } - - List ParseDescrCodeBlock(TokenRange toks) - { - List declarations = new List(); - while (true) - { - Token delim = toks.FirstOrDefault(t => t.Value == ";"); - if (delim == null) - break; - - int pos = delim.Position + 1; - var potentialComment = range(pos, toks.End).SkipWhile(t => t.Value == "\t" || t.Value == " "); - if (!potentialComment.IsEmpty && potentialComment.First().Value.StartsWith("//")) - { - pos = potentialComment.First().Position + 1; - } - - ParseDeclaration(range(toks.Begin, pos), declarations); - - toks = range(pos, toks.End); - } - if (!toks.All(Whitespace)) - throw new Error(toks.First(NonWhitespace).SourceLine, "Trailing unterminated statement in code block"); - return declarations; - } - - CodeBlock ParseCodeBlock(TokenRange toks) - { - List statements = new List(); - while (true) - { - Token delim = toks - .FirstOrDefault( - t=> t.ParenDepth == toks.First().ParenDepth && - t.BraceDepth == toks.First().BraceDepth && - (t.Value==";" || t.Value == "}") - ); - if (delim == null) - break; - ParseStatement(range(toks.Begin, delim.Position + 1), statements); - toks = range(delim.Position + 1, toks.End); - } - if (!toks.All(Whitespace)) - throw new Error(toks.First(NonWhitespace).SourceLine, "Trailing unterminated statement in code block"); - return new CodeBlock { statements = statements.ToArray() }; - } - - TokenRange range(int beginPos, int endPos) - { - return new TokenRange(tokens, beginPos, endPos); - } - - Descr ParseDescr(int pos, out int end) - { - var descr = new Descr(); - var toks = range(pos + 1, tokens.Length); - var heading = toks.TakeWhile(t => t.Value != "{"); - var body = range(heading.End + 1, tokens.Length) - .TakeWhile(t => t.BraceDepth > toks.First().BraceDepth || t.Value == ";" ); //assumes no whitespace between the last "}" and the ";" - - ParseDescrHeading(descr, heading); - descr.body = ParseDescrCodeBlock(body); - - end = body.End + 1; - return descr; - } - - Actor ParseActor( int pos, out int end ) { - var actor = new Actor(); - var head_token = tokens[pos]; - actor.SourceLine = head_token.SourceLine; - - var toks = range(pos+1, tokens.Length); - var heading = toks.TakeWhile(t => t.Value != "{"); - var toSemicolon = toks.TakeWhile(t => t.Value != ";"); - actor.isForwardDeclaration = toSemicolon.Length < heading.Length; - if (actor.isForwardDeclaration) { - heading = toSemicolon; - if (head_token.Value == "ACTOR" || head_token.Value == "SWIFT_ACTOR") { - ParseActorHeading(actor, heading); - } else { - head_token.Assert("ACTOR expected!", t => false); - } - end = heading.End + 1; - } else { - var body = range(heading.End+1, tokens.Length) - .TakeWhile(t => t.BraceDepth > toks.First().BraceDepth); - - if (head_token.Value == "ACTOR" || head_token.Value == "SWIFT_ACTOR") - { - ParseActorHeading(actor, heading); - } - else if (head_token.Value == "TEST_CASE") { - ParseTestCaseHeading(actor, heading); - actor.isTestCase = true; - } - else - head_token.Assert("ACTOR or TEST_CASE expected!", t => false); - - actor.body = ParseCodeBlock(body); - - if (!actor.body.containsWait()) - this.errorMessagePolicy.HandleActorWithoutWait(sourceFile, actor); - - end = body.End + 1; - } - return actor; - } - - string str(IEnumerable tokens) - { - return string.Join("", tokens.Select(x => x.Value).ToArray()); - } - string str(int begin, int end) - { - return str(range(begin,end)); - } - - void CountParens() - { - int BraceDepth = 0, ParenDepth = 0, LineCount = 1; - Token lastParen = null, lastBrace = null; - for (int i = 0; i < tokens.Length; i++) - { - switch (tokens[i].Value) - { - case "}": BraceDepth--; break; - case ")": ParenDepth--; break; - case "\r\n": LineCount++; break; - case "\n": LineCount++; break; - } - if (BraceDepth < 0) throw new Error(LineCount, "Mismatched braces"); - if (ParenDepth < 0) throw new Error(LineCount, "Mismatched parenthesis"); - tokens[i].Position = i; - tokens[i].SourceLine = LineCount; - tokens[i].BraceDepth = BraceDepth; - tokens[i].ParenDepth = ParenDepth; - if (tokens[i].Value.StartsWith("/*")) LineCount += tokens[i].Value.Count(c=>c=='\n'); - switch (tokens[i].Value) - { - case "{": BraceDepth++; if (BraceDepth==1) lastBrace = tokens[i]; break; - case "(": ParenDepth++; if (ParenDepth==1) lastParen = tokens[i]; break; - } - } - if (BraceDepth != 0) throw new Error(lastBrace.SourceLine, "Unmatched brace"); - if (ParenDepth != 0) throw new Error(lastParen.SourceLine, "Unmatched parenthesis"); - } - - void showTokens() - { - foreach (var t in tokens) - { - if (t.Value == "\r\n") - Console.WriteLine(); - else if (t.Value.Length == 1) - Console.Write(t.Value); - else - Console.Write("|{0}|", t.Value); - } - } - - readonly Regex identifierPattern = new Regex(@"\G[a-zA-Z_][a-zA-Z_0-9]*", RegexOptions.Singleline); - - readonly Regex[] tokenExpressions = (new string[] { - @"\{", - @"\}", - @"\(", - @"\)", - @"\[", - @"\]", - @"//[^\n]*", - @"/[*]([*][^/]|[^*])*[*]/", - @"'(\\.|[^\'\n])*'", //< SOMEDAY: Not fully restrictive - @"""(\\.|[^\""\n])*""", - @"[a-zA-Z_][a-zA-Z_0-9]*", - @"\r\n", - @"\n", - @"::", - @":", - @"#[a-z]*", // Recognize preprocessor directives so that we can reject them - @".", - }).Select( x=>new Regex(@"\G"+x, RegexOptions.Singleline) ).ToArray(); - - IEnumerable Tokenize(string text) - { - int pos = 0; - while (pos < text.Length) - { - bool ok = false; - foreach (var re in tokenExpressions) - { - var m = re.Match(text, pos); - if (m.Success) - { - yield return m.Value; - pos += m.Value.Length; - ok = true; - break; - } - } - if (!ok) - throw new Exception( String.Format("Can't tokenize! {0}", pos)); - } - } - } -} diff --git a/flow/actorcompiler/ParseTree.cs b/flow/actorcompiler/ParseTree.cs deleted file mode 100644 index 766d56b304c..00000000000 --- a/flow/actorcompiler/ParseTree.cs +++ /dev/null @@ -1,252 +0,0 @@ -/* - * ParseTree.cs - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; - -namespace actorcompiler -{ - class VarDeclaration - { - public string type; - public string name; - public string initializer; - public bool initializerConstructorSyntax; - }; - - abstract class Statement - { - public int FirstSourceLine; - public virtual bool containsWait() - { - return false; - } - }; - class PlainOldCodeStatement : Statement - { - public string code; - public override string ToString() - { - return code; - } - }; - class StateDeclarationStatement : Statement - { - public VarDeclaration decl; - public override string ToString() - { - if (decl.initializerConstructorSyntax) - return string.Format("State {0} {1}({2});", decl.type, decl.name, decl.initializer); - else - return string.Format("State {0} {1} = {2};", decl.type, decl.name, decl.initializer); - } - }; - class WhileStatement : Statement - { - public string expression; - public Statement body; - public override bool containsWait() - { - return body.containsWait(); - } - }; - class ForStatement : Statement - { - public string initExpression = ""; - public string condExpression = ""; - public string nextExpression = ""; - public Statement body; - public override bool containsWait() - { - return body.containsWait(); - } - }; - class RangeForStatement : Statement - { - public string rangeExpression; - public string rangeDecl; - public Statement body; - public override bool containsWait() - { - return body.containsWait(); - } - }; - class LoopStatement : Statement - { - public Statement body; - public override string ToString() - { - return "Loop " + body.ToString(); - } - public override bool containsWait() - { - return body.containsWait(); - } - }; - class BreakStatement : Statement - { - }; - class ContinueStatement : Statement - { - }; - class IfStatement : Statement - { - public string expression; - public bool constexpr; - public Statement ifBody; - public Statement elseBody; // might be null - public override bool containsWait() - { - return ifBody.containsWait() || (elseBody != null && elseBody.containsWait()); - } - }; - class ReturnStatement : Statement - { - public string expression; - public override string ToString() - { - return "Return " + expression; - } - }; - class WaitStatement : Statement - { - public VarDeclaration result; - public string futureExpression; - public bool resultIsState; - public bool isWaitNext; - public override string ToString() - { - return string.Format("Wait {0} {1} <- {2} ({3})", result.type, result.name, futureExpression, resultIsState ? "state" : "local"); - } - public override bool containsWait() - { - return true; - } - }; - class ChooseStatement : Statement - { - public Statement body; - public override string ToString() - { - return "Choose " + body.ToString(); - } - public override bool containsWait() - { - return body.containsWait(); - } - }; - class WhenStatement : Statement - { - public WaitStatement wait; - public Statement body; - public override string ToString() - { - return string.Format("When ({0}) {1}", wait, body); - } - public override bool containsWait() - { - return true; - } - }; - class TryStatement : Statement - { - public struct Catch - { - public string expression; - public Statement body; - public int FirstSourceLine; - }; - - public Statement tryBody; - public List catches; - public override bool containsWait() - { - if (tryBody.containsWait()) - return true; - foreach (Catch c in catches) - if (c.body.containsWait()) - return true; - return false; - } - }; - class ThrowStatement : Statement - { - public string expression; - }; - - class CodeBlock : Statement - { - public Statement[] statements; - public override string ToString() - { - return string.Join("\n", - new string[] { "CodeBlock" } - .Concat(statements.Select(s => s.ToString())) - .Concat(new string[] { "EndCodeBlock" }) - .ToArray()); - } - public override bool containsWait() - { - foreach (Statement s in statements) - if (s.containsWait()) - return true; - return false; - } - }; - - - class Declaration - { - public string type; - public string name; - public string comment; - }; - - class Actor - { - public List attributes = new List(); - public string returnType; - public string name; - public string enclosingClass = null; - public VarDeclaration[] parameters; - public VarDeclaration[] templateFormals; //< null if not a template - public CodeBlock body; - public int SourceLine; - public bool isStatic = false; - private bool isUncancellable; - public string testCaseParameters = null; - public string nameSpace = null; - public bool isForwardDeclaration = false; - public bool isTestCase = false; - - public bool IsCancellable() { return returnType != null && !isUncancellable; } - public void SetUncancellable() { isUncancellable = true; } - }; - - class Descr - { - public string name; - public string superClassList; - public List body; - }; -}; diff --git a/flow/actorcompiler/Program.cs b/flow/actorcompiler/Program.cs deleted file mode 100644 index 742dd21a24f..00000000000 --- a/flow/actorcompiler/Program.cs +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Program.cs - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.IO; - -namespace actorcompiler -{ - class Program - { - private static void OverwriteByMove(string target, string temporaryFile) { - if (File.Exists(target)) - { - File.SetAttributes(target, FileAttributes.Normal); - File.Delete(target); - } - File.Move(temporaryFile, target); - File.SetAttributes(target, FileAttributes.ReadOnly); - } - - public static int Main(string[] args) - { - bool generateProbes = false; - if (args.Length < 2) - { - Console.WriteLine("Usage:"); - Console.WriteLine(" actorcompiler [--disable-diagnostics] [--generate-probes]"); - return 100; - } - Console.WriteLine("actorcompiler {0}", string.Join(" ", args)); - string input = args[0], output = args[1], outputtmp = args[1] + ".tmp", outputUid = args[1] + ".uid"; - ErrorMessagePolicy errorMessagePolicy = new ErrorMessagePolicy(); - foreach (var arg in args) { - if (arg.StartsWith("--")) { - if (arg.Equals("--disable-diagnostics")) { - errorMessagePolicy.DisableDiagnostics = true; - } else if (arg.Equals("--generate-probes")) { - generateProbes = true; - } - } - } - try - { - var inputData = File.ReadAllText(input); - var parser = new ActorParser(inputData, input.Replace('\\', '/'), errorMessagePolicy, generateProbes); - - using (var outputStream = new StreamWriter(outputtmp)) { - parser.Write(outputStream, output.Replace('\\', '/')); - } - OverwriteByMove(output, outputtmp); - - using (var outputStream = new StreamWriter(outputtmp)) { - foreach(var entry in parser.uidObjects) { - outputStream.WriteLine("{0}|{1}|{2}", entry.Key.Item1, entry.Key.Item2, entry.Value); - } - } - OverwriteByMove(outputUid, outputtmp); - - return 0; - } - catch (actorcompiler.Error e) - { - Console.Error.WriteLine("{0}({1}): error FAC1000: {2}", input, e.SourceLine, e.Message); - if (File.Exists(outputtmp)) - File.Delete(outputtmp); - if (File.Exists(output)) - { - File.SetAttributes(output, FileAttributes.Normal); - File.Delete(output); - } - return 1; - } - catch (Exception e) - { - Console.Error.WriteLine("{0}({1}): error FAC2000: Internal {2}", input, 1, e.ToString()); - if (File.Exists(outputtmp)) - File.Delete(outputtmp); - if (File.Exists(output)) - { - File.SetAttributes(output, FileAttributes.Normal); - File.Delete(output); - } - return 3; - } - } - } -} diff --git a/flow/actorcompiler/Properties/AssemblyInfo.cs b/flow/actorcompiler/Properties/AssemblyInfo.cs deleted file mode 100644 index bec6889f4b5..00000000000 --- a/flow/actorcompiler/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,56 +0,0 @@ -/* - * AssemblyInfo.cs - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("actorcompiler")] -[assembly: AssemblyDescription("Compile Flow code to C++")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("Apple Inc")] -[assembly: AssemblyProduct("actorcompiler")] -[assembly: AssemblyCopyright("Copyright (c) 2013-2025 Apple Inc.")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("fd8d34c8-46ca-43de-bfb3-b437e46943da")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/flow/actorcompiler/__init__.py b/flow/actorcompiler/__init__.py new file mode 100644 index 00000000000..71a1db4a044 --- /dev/null +++ b/flow/actorcompiler/__init__.py @@ -0,0 +1,33 @@ +""" +Python port of the Flow actor compiler. + +Typical usage: + from flow.actorcompiler import main + main(["input.actor.cpp", "output.g.actor.cpp"]) +""" + + +class ActorCompilerError(Exception): + """Exception raised for parser or compiler errors with source locations.""" + + def __init__(self, source_line: int, message: str, *args: object) -> None: + if args: + message = message.format(*args) + super().__init__(message) + self.source_line = source_line + + def __str__(self) -> str: + return f"{super().__str__()} (line {self.source_line})" + + +from .actor_compiler import ActorCompiler +from .actor_parser import ActorParser, ErrorMessagePolicy +from .main import main + +__all__ = [ + "ActorCompiler", + "ActorParser", + "ErrorMessagePolicy", + "ActorCompilerError", + "main", +] diff --git a/flow/actorcompiler/__main__.py b/flow/actorcompiler/__main__.py new file mode 100644 index 00000000000..0effc52441c --- /dev/null +++ b/flow/actorcompiler/__main__.py @@ -0,0 +1,10 @@ +""" +Allow running the actorcompiler as a module: + python3 -m flow.actorcompiler input.actor.cpp output.g.cpp +""" + +import sys +from .main import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/flow/actorcompiler/actor_compiler.py b/flow/actorcompiler/actor_compiler.py new file mode 100644 index 00000000000..7ac4c5603c2 --- /dev/null +++ b/flow/actorcompiler/actor_compiler.py @@ -0,0 +1,1583 @@ +from __future__ import annotations + +import hashlib +import io +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + +from . import ActorCompilerError +from .parse_tree import ( + Actor, + BreakStatement, + ChooseStatement, + CodeBlock, + ContinueStatement, + Declaration, + Descr, + ForStatement, + IfStatement, + LoopStatement, + PlainOldCodeStatement, + RangeForStatement, + ReturnStatement, + Statement, + StateDeclarationStatement, + ThrowStatement, + TryStatement, + VarDeclaration, + WaitStatement, + WhenStatement, + WhileStatement, +) + + +class Function: + def __init__( + self, + name: str = "", + return_type: str = "int", + formal_parameters: Optional[Sequence[str]] = None, + ) -> None: + self.name = name + self.returnType = return_type + self.formalParameters = list(formal_parameters or []) + self.endIsUnreachable = False + self.exceptionParameterIs: Optional[str] = None + self.publicName = False + self.specifiers = "" + self._indentation = "" + self._body = io.StringIO() + self.wasCalled = False + self.overload: Optional["Function"] = None + + def set_overload(self, overload: "Function") -> None: + self.overload = overload + + def pop_overload(self) -> Optional["Function"]: + overload = self.overload + self.overload = None + return overload + + def add_overload(self, *formal_parameters: str) -> None: + overload = Function( + name=self.name, + return_type=self.returnType, + formal_parameters=formal_parameters, + ) + overload.endIsUnreachable = self.endIsUnreachable + overload._indentation = self._indentation + self.set_overload(overload) + + def indent(self, change: int) -> None: + if change > 0: + self._indentation += "\t" * change + elif change < 0: + self._indentation = self._indentation[:change] + if self.overload is not None: + self.overload.indent(change) + + def write_line_unindented(self, text: str) -> None: + self._body.write(f"{text}\n") + if self.overload is not None: + self.overload.write_line_unindented(text) + + def write_line(self, text: str, *args: object) -> None: + if args: + text = text.format(*args) + self._body.write(f"{self._indentation}{text}\n") + if self.overload is not None: + self.overload.write_line(text) + + @property + def body_text(self) -> str: + value = self._body.getvalue() + if self.overload is not None: + _ = self.overload.body_text + return value + + def use_by_name(self) -> str: + self.wasCalled = True + return self.name if self.publicName else f"a_{self.name}" + + def call(self, *parameters: str) -> str: + params = ", ".join(parameters) + return f"{self.use_by_name()}({params})" + + +class LiteralBreak(Function): + def __init__(self) -> None: + super().__init__(name="break!", return_type="") + + def call(self, *parameters: str) -> str: + if parameters: + raise ActorCompilerError(0, "LiteralBreak called with parameters!") + self.wasCalled = True + return "break" + + +class LiteralContinue(Function): + def __init__(self) -> None: + super().__init__(name="continue!", return_type="") + + def call(self, *parameters: str) -> str: + if parameters: + raise ActorCompilerError(0, "LiteralContinue called with parameters!") + self.wasCalled = True + return "continue" + + +@dataclass +class StateVar: + type: str = "" + name: str = "" + initializer: Optional[str] = None + initializerConstructorSyntax: bool = False + SourceLine: int = 0 + + +@dataclass +class CallbackVar: + type: str = "" + CallbackGroup: int = 0 + SourceLine: int = 0 + + +class Context: + def __init__( + self, + target: Optional[Function] = None, + next_func: Optional[Function] = None, + breakF: Optional[Function] = None, + continueF: Optional[Function] = None, + catchFErr: Optional[Function] = None, + tryLoopDepth: int = 0, + ) -> None: + self.target = target + self.next = next_func + self.breakF = breakF + self.continueF = continueF + self.catchFErr = catchFErr + self.tryLoopDepth = tryLoopDepth + + def unreachable(self) -> None: + self.target = None + + def with_target(self, new_target: Function) -> "Context": + return Context( + target=new_target, + breakF=self.breakF, + continueF=self.continueF, + catchFErr=self.catchFErr, + tryLoopDepth=self.tryLoopDepth, + ) + + def loop_context( + self, + new_target: Function, + breakF: Function, + continueF: Function, + deltaLoopDepth: int, + ) -> "Context": + return Context( + target=new_target, + breakF=breakF, + continueF=continueF, + catchFErr=self.catchFErr, + tryLoopDepth=self.tryLoopDepth + deltaLoopDepth, + ) + + def with_catch(self, new_catch: Function) -> "Context": + return Context( + target=self.target, + breakF=self.breakF, + continueF=self.continueF, + catchFErr=new_catch, + ) + + def clone(self) -> "Context": + return Context( + target=self.target, + next_func=self.next, + breakF=self.breakF, + continueF=self.continueF, + catchFErr=self.catchFErr, + tryLoopDepth=self.tryLoopDepth, + ) + + def copy_from(self, other: "Context") -> None: + self.target = other.target + self.next = other.next + self.breakF = other.breakF + self.continueF = other.continueF + self.catchFErr = other.catchFErr + self.tryLoopDepth = other.tryLoopDepth + + +class DescrCompiler: + def __init__(self, descr: "Descr", brace_depth: int) -> None: + self.descr = descr + self.member_indent_str = "\t" * brace_depth + + def write(self, writer) -> int: + lines = 0 + indent = self.member_indent_str + writer.write( + f"{indent}template<> struct Descriptor {{\n" + ) + writer.write( + f'{indent}\tstatic StringRef typeName() {{ return "{self.descr.name}"_sr; }}\n' + ) + writer.write(f"{indent}\ttypedef {self.descr.name} type;\n") + lines += 3 + for dec in self.descr.body: + writer.write(f"{indent}\tstruct {dec.name}Descriptor {{\n") + writer.write( + f'{indent}\t\tstatic StringRef name() {{ return "{dec.name}"_sr; }}\n' + ) + writer.write( + f'{indent}\t\tstatic StringRef typeName() {{ return "{dec.type}"_sr; }}\n' + ) + writer.write( + f'{indent}\t\tstatic StringRef comment() {{ return "{dec.comment}"_sr; }}\n' + ) + writer.write(f"{indent}\t\ttypedef {dec.type} type;\n") + writer.write( + f"{indent}\t\tstatic inline type get({self.descr.name}& from);\n" + ) + writer.write(f"{indent}\t}};\n") + lines += 7 + writer.write(f"{indent}\ttypedef std::tuple<") + first = True + for dec in self.descr.body: + if not first: + writer.write(",") + writer.write(f"{dec.name}Descriptor") + first = False + writer.write("> fields;\n") + writer.write( + f"{indent}\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type field_indexes;\n" + ) + writer.write(f"{indent}}};\n") + if self.descr.superClassList: + writer.write( + f"{indent}struct {self.descr.name} : {self.descr.superClassList} {{\n" + ) + else: + writer.write(f"{indent}struct {self.descr.name} {{\n") + lines += 4 + for dec in self.descr.body: + writer.write(f"{indent}\t{dec.type} {dec.name}; //{dec.comment}\n") + lines += 1 + writer.write(f"{indent}}};\n") + lines += 1 + for dec in self.descr.body: + writer.write( + f"{indent}{dec.type} Descriptor<{self.descr.name}>::{dec.name}Descriptor::get({self.descr.name}& from) {{ return from.{dec.name}; }}\n" + ) + lines += 1 + return lines + + +class ActorCompiler: + member_indent_str = "\t" + loopDepth0 = "int loopDepth=0" + loopDepth = "int loopDepth" + codeIndent = 2 + usedClassNames: set[str] = set() + + def __init__( + self, + actor: Actor, + source_file: str, + is_top_level: bool, + line_numbers_enabled: bool, + generate_probes: bool, + ) -> None: + self.actor = actor + self.sourceFile = source_file + self.isTopLevel = is_top_level + self.LineNumbersEnabled = line_numbers_enabled + self.generateProbes = generate_probes + self.className = "" + self.fullClassName = "" + self.stateClassName = "" + self.state: List[StateVar] = [] + self.callbacks: List[CallbackVar] = [] + self.chooseGroups = 0 + self.whenCount = 0 + self.This = "" + self.uidObjects: Dict[Tuple[int, int], str] = {} + self.functions: Dict[str, Function] = {} + self.iterators: Dict[str, int] = {} + self.FindState() + + def ByteToLong(self, data: bytes) -> int: + result = 0 + for b in data: + result += b + result <<= 8 + return result & ((1 << 64) - 1) + + def GetUidFromString(self, value: str) -> Tuple[int, int]: + digest = hashlib.sha256(value.encode("utf-8")).digest() + first = self.ByteToLong(digest[:8]) + second = self.ByteToLong(digest[8:16]) + return (first, second) + + def WriteActorFunction(self, writer, full_return_type: str) -> None: + self.WriteTemplate(writer) + self.LineNumber(writer, self.actor.SourceLine) + for attribute in self.actor.attributes: + writer.write(f"{attribute} ") + if self.actor.isStatic: + writer.write("static ") + namespace_prefix = ( + "" if self.actor.nameSpace is None else f"{self.actor.nameSpace}::" + ) + params = ", ".join(self.ParameterList()) + writer.write( + f"{full_return_type} {namespace_prefix}{self.actor.name}( {params} ) {{\n" + ) + self.LineNumber(writer, self.actor.SourceLine) + ctor_args = ", ".join(param.name for param in self.actor.parameters) + new_actor = f"new {self.fullClassName}({ctor_args})" + if self.actor.returnType is not None: + writer.write(f"\treturn Future<{self.actor.returnType}>({new_actor});\n") + else: + writer.write(f"\t{new_actor};\n") + writer.write("}\n") + + def WriteActorClass( + self, writer, full_state_class_name: str, body: Function + ) -> None: + writer.write( + f"// This generated class is to be used only via {self.actor.name}()\n" + ) + self.WriteTemplate(writer) + self.LineNumber(writer, self.actor.SourceLine) + callback_bases = ", ".join(f"public {cb.type}" for cb in self.callbacks) + if callback_bases: + callback_bases += ", " + writer.write( + "class {0} final : public Actor<{2}>, {3}public FastAllocated<{1}>, public {4} {{\n".format( + self.className, + self.fullClassName, + "void" if self.actor.returnType is None else self.actor.returnType, + callback_bases, + full_state_class_name, + ) + ) + writer.write("public:\n") + writer.write(f"\tusing FastAllocated<{self.fullClassName}>::operator new;\n") + writer.write(f"\tusing FastAllocated<{self.fullClassName}>::operator delete;\n") + actor_identifier_key = f"{self.sourceFile}:{self.actor.name}" + uid = self.GetUidFromString(actor_identifier_key) + self.uidObjects[(uid[0], uid[1])] = actor_identifier_key + writer.write( + f"\tstatic constexpr ActorIdentifier __actorIdentifier = UID({uid[0]}UL, {uid[1]}UL);\n" + ) + writer.write("\tActiveActorHelper activeActorHelper;\n") + writer.write("#pragma clang diagnostic push\n") + writer.write('#pragma clang diagnostic ignored "-Wdelete-non-virtual-dtor"\n') + if self.actor.returnType is not None: + writer.write(" void destroy() override {\n") + writer.write(" activeActorHelper.~ActiveActorHelper();\n") + writer.write( + f" static_cast*>(this)->~Actor();\n" + ) + writer.write(" operator delete(this);\n") + writer.write(" }\n") + else: + writer.write(" void destroy() {{\n") + writer.write(" activeActorHelper.~ActiveActorHelper();\n") + writer.write(" static_cast*>(this)->~Actor();\n") + writer.write(" operator delete(this);\n") + writer.write(" }}\n") + writer.write("#pragma clang diagnostic pop\n") + for cb in self.callbacks: + writer.write(f"friend struct {cb.type};\n") + self.LineNumber(writer, self.actor.SourceLine) + self.WriteConstructor(body, writer, full_state_class_name) + self.WriteCancelFunc(writer) + writer.write("};\n") + + def Write(self, writer) -> None: + full_return_type = ( + f"Future<{self.actor.returnType}>" + if self.actor.returnType is not None + else "void" + ) + for i in range(1 << 16): + class_name = "{3}{0}{1}Actor{2}".format( + self.actor.name[:1].upper(), + self.actor.name[1:], + str(i) if i != 0 else "", + ( + self.actor.enclosingClass.replace("::", "_") + "_" + if self.actor.enclosingClass is not None + and self.actor.isForwardDeclaration + else ( + self.actor.nameSpace.replace("::", "_") + "_" + if self.actor.nameSpace is not None + else "" + ) + ), + ) + if self.actor.isForwardDeclaration: + self.className = class_name + break + if class_name not in ActorCompiler.usedClassNames: + self.className = class_name + ActorCompiler.usedClassNames.add(class_name) + break + self.fullClassName = self.className + self.GetTemplateActuals() + actor_class_formal = VarDeclaration(name=self.className, type="class") + self.This = f"static_cast<{actor_class_formal.name}*>(this)" + self.stateClassName = self.className + "State" + full_state = self.stateClassName + self.GetTemplateActuals( + VarDeclaration(type="class", name=self.fullClassName) + ) + if self.actor.isForwardDeclaration: + for attribute in self.actor.attributes: + writer.write(f"{attribute} ") + if self.actor.isStatic: + writer.write("static ") + ns = "" if self.actor.nameSpace is None else f"{self.actor.nameSpace}::" + writer.write( + f"{full_return_type} {ns}{self.actor.name}( {', '.join(self.ParameterList())} );\n" + ) + if self.actor.enclosingClass is not None: + writer.write(f"template friend class {self.stateClassName};\n") + return + body = self.getFunction("", "body", self.loopDepth0) + body_context = Context( + target=body, + catchFErr=self.getFunction( + body.name, "Catch", "Error error", self.loopDepth0 + ), + ) + end_context = self.TryCatchCompile(self.actor.body, body_context) + if end_context.target is not None: + if self.actor.returnType is None: + self.compile_statement( + ReturnStatement( + FirstSourceLine=self.actor.SourceLine, expression="" + ), + end_context, + ) + else: + raise ActorCompilerError( + self.actor.SourceLine, + "Actor {0} fails to return a value", + self.actor.name, + ) + if self.actor.returnType is not None: + body_context.catchFErr.write_line(f"this->~{self.stateClassName}();") + body_context.catchFErr.write_line( + f"{self.This}->sendErrorAndDelPromiseRef(error);" + ) + else: + body_context.catchFErr.write_line(f"delete {self.This};") + body_context.catchFErr.write_line("loopDepth = 0;") + if self.isTopLevel and self.actor.nameSpace is None: + writer.write("namespace {\n") + writer.write( + f"// This generated class is to be used only via {self.actor.name}()\n" + ) + self.WriteTemplate(writer, actor_class_formal) + self.LineNumber(writer, self.actor.SourceLine) + writer.write(f"class {self.stateClassName} {{\n") + writer.write("public:\n") + self.LineNumber(writer, self.actor.SourceLine) + self.WriteStateConstructor(writer) + self.WriteStateDestructor(writer) + self.WriteFunctions(writer) + for st in self.state: + self.LineNumber(writer, st.SourceLine) + writer.write(f"\t{st.type} {st.name};\n") + writer.write("};\n") + self.WriteActorClass(writer, full_state, body) + if self.isTopLevel and self.actor.nameSpace is None: + writer.write("} // namespace\n") + self.WriteActorFunction(writer, full_return_type) + if self.actor.testCaseParameters is not None: + writer.write( + f"ACTOR_TEST_CASE({self.actor.name}, {self.actor.testCaseParameters})\n" + ) + + thisAddress = "reinterpret_cast(this)" + + def ProbeEnter(self, fun: Function, name: str, index: int = -1) -> None: + if self.generateProbes: + fun.write_line( + 'fdb_probe_actor_enter("{0}", {1}, {2});', + name, + self.thisAddress, + index, + ) + block_identifier = self.GetUidFromString(fun.name) + fun.write_line("#ifdef WITH_ACAC") + fun.write_line( + "static constexpr ActorBlockIdentifier __identifier = UID({0}UL, {1}UL);", + block_identifier[0], + block_identifier[1], + ) + fun.write_line( + "ActorExecutionContextHelper __helper(static_cast<{0}*>(this)->activeActorHelper.actorID, __identifier);", + self.className, + ) + fun.write_line("#endif // WITH_ACAC") + + def ProbeExit(self, fun: Function, name: str, index: int = -1) -> None: + if self.generateProbes: + fun.write_line( + 'fdb_probe_actor_exit("{0}", {1}, {2});', + name, + self.thisAddress, + index, + ) + + def ProbeCreate(self, fun: Function, name: str) -> None: + if self.generateProbes: + fun.write_line( + 'fdb_probe_actor_create("{0}", {1});', name, self.thisAddress + ) + + def ProbeDestroy(self, fun: Function, name: str) -> None: + if self.generateProbes: + fun.write_line( + 'fdb_probe_actor_destroy("{0}", {1});', + name, + self.thisAddress, + ) + + def LineNumber(self, writer, source_line: int) -> None: + if source_line == 0: + raise ActorCompilerError(0, "Invalid source line (0)") + if self.LineNumbersEnabled: + writer.write( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( + source_line, self.sourceFile + ) + ) + + def LineNumberFunction(self, func: Function, source_line: int) -> None: + if source_line == 0: + raise ActorCompilerError(0, "Invalid source line (0)") + if self.LineNumbersEnabled: + func.write_line_unindented( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"'.format( + source_line, self.sourceFile + ) + ) + + def TryCatch( + self, + cx: Context, + catch_function: Optional[Function], + catch_loop_depth: int, + action, + use_loop_depth: bool = True, + ) -> None: + if catch_function is not None: + cx.target.write_line("try {") + cx.target.indent(+1) + action() + if catch_function is not None: + cx.target.indent(-1) + cx.target.write_line("}") + cx.target.write_line("catch (Error& error) {") + if use_loop_depth: + cx.target.write_line( + "\tloopDepth = {0};", + catch_function.call( + "error", self.AdjustLoopDepth(catch_loop_depth) + ), + ) + else: + cx.target.write_line("\t{0};", catch_function.call("error", "0")) + cx.target.write_line("} catch (...) {") + if use_loop_depth: + cx.target.write_line( + "\tloopDepth = {0};", + catch_function.call( + "unknown_error()", self.AdjustLoopDepth(catch_loop_depth) + ), + ) + else: + cx.target.write_line( + "\t{0};", catch_function.call("unknown_error()", "0") + ) + cx.target.write_line("}") + + def TryCatchCompile(self, block: CodeBlock, cx: Context) -> Context: + result_holder = {"ctx": cx} + + def action() -> None: + result_holder["ctx"] = self._try_catch_compile_body(block, cx) + + self.TryCatch(cx, cx.catchFErr, cx.tryLoopDepth, action) + return result_holder["ctx"] + + def _try_catch_compile_body(self, block: CodeBlock, cx: Context) -> Context: + compiled = self.Compile(block, cx, True) + if compiled.target is not None: + next_func = self.getFunction(compiled.target.name, "cont", self.loopDepth) + compiled.target.write_line("loopDepth = {0};", next_func.call("loopDepth")) + compiled.target = next_func + compiled.next = None + return compiled + + def WriteTemplate(self, writer, *extra_parameters: VarDeclaration) -> None: + formals = list(self.actor.templateFormals or []) + list(extra_parameters) + if not formals: + return + self.LineNumber(writer, self.actor.SourceLine) + writer.write( + "template <{0}>\n".format(", ".join(f"{p.type} {p.name}" for p in formals)) + ) + + def GetTemplateActuals(self, *extra_parameters: VarDeclaration) -> str: + formals = list(self.actor.templateFormals or []) + list(extra_parameters) + if not formals: + return "" + return "<{0}>".format(", ".join(p.name for p in formals)) + + def WillContinue(self, stmt: Statement) -> bool: + return any( + isinstance(sub, (ChooseStatement, WaitStatement, TryStatement)) + for sub in self.Flatten(stmt) + ) + + def AsCodeBlock(self, statement: Statement) -> CodeBlock: + if isinstance(statement, CodeBlock): + return statement + return CodeBlock(statements=[statement]) + + def Flatten(self, stmt: Optional[Statement]) -> Iterable[Statement]: + if stmt is None: + return [] + + def _flatten(s: Statement) -> Iterable[Statement]: + yield s + if isinstance(s, LoopStatement): + yield from self.Flatten(s.body) + elif isinstance(s, WhileStatement): + yield from self.Flatten(s.body) + elif isinstance(s, ForStatement): + yield from self.Flatten(s.body) + elif isinstance(s, RangeForStatement): + yield from self.Flatten(s.body) + elif isinstance(s, CodeBlock): + for child in s.statements: + yield from self.Flatten(child) + elif isinstance(s, IfStatement): + yield from self.Flatten(s.ifBody) + if s.elseBody: + yield from self.Flatten(s.elseBody) + elif isinstance(s, ChooseStatement): + yield from self.Flatten(s.body) + elif isinstance(s, WhenStatement): + if s.body: + yield from self.Flatten(s.body) + elif isinstance(s, TryStatement): + yield from self.Flatten(s.tryBody) + for c in s.catches: + yield from self.Flatten(c.body) + + return list(_flatten(stmt)) + + def FindState(self) -> None: + self.state = [ + StateVar( + SourceLine=self.actor.SourceLine, + name=p.name, + type=p.type, + initializer=p.name, + ) + for p in self.actor.parameters + ] + + def AdjustLoopDepth(self, subtract: int) -> str: + if subtract == 0: + return "loopDepth" + return f"std::max(0, loopDepth - {subtract})" + + def ParameterList(self) -> List[str]: + params = [] + for p in self.actor.parameters: + if p.initializer: + params.append(f"{p.type} const& {p.name} = {p.initializer}") + else: + params.append(f"{p.type} const& {p.name}") + return params + + def getFunction( + self, + base_name: str, + add_name: str, + *formal_parameters: str, + overload_formal_parameters: Optional[Sequence[str]] = None, + ) -> Function: + if len(formal_parameters) == 1 and isinstance( + formal_parameters[0], (list, tuple) + ): + params = list(formal_parameters[0]) + else: + params = [p for p in formal_parameters if p] + + if add_name == "cont" and len(base_name) >= 5 and base_name[-5:-1] == "cont": + proposed_name = base_name[:-1] + else: + proposed_name = base_name + add_name + idx = 1 + while f"{proposed_name}{idx}" in self.functions: + idx += 1 + func = Function( + name=f"{proposed_name}{idx}", + return_type="int", + formal_parameters=params, + ) + func.indent(self.codeIndent) + if overload_formal_parameters: + func.add_overload(*overload_formal_parameters) + self.functions[func.name] = func + return func + + def WriteFunctions(self, writer) -> None: + for func in self.functions.values(): + body = func.body_text + if body: + self.WriteFunction(writer, func, body) + if func.overload: + overload_body = func.overload.body_text + if overload_body: + self.WriteFunction(writer, func.overload, overload_body) + + def WriteFunction(self, writer, func: Function, body: str) -> None: + spec = "" if not func.specifiers else f" {func.specifiers}" + trailing = " " if spec == "" else "" + signature = ( + f"{self.member_indent_str}" + f"{'' if func.returnType == '' else f'{func.returnType} '}" + f"{func.use_by_name()}({','.join(func.formalParameters)})" + f"{spec}{trailing}\n" + ) + writer.write(signature) + if func.returnType != "": + writer.write(f"{self.member_indent_str}{{\n") + writer.write(body) + writer.write("\n") + if not func.endIsUnreachable: + writer.write(f"{self.member_indent_str}\treturn loopDepth;\n") + writer.write(f"{self.member_indent_str}}}\n") + + def WriteCancelFunc(self, writer) -> None: + if not self.actor.IsCancellable(): + return + cancel_func = Function( + name="cancel", + return_type="void", + formal_parameters=[], + ) + cancel_func.endIsUnreachable = True + cancel_func.publicName = True + cancel_func.specifiers = "override" + cancel_func.indent(self.codeIndent) + cancel_func.write_line("auto wait_state = this->actor_wait_state;") + cancel_func.write_line("this->actor_wait_state = -1;") + cancel_func.write_line("switch (wait_state) {") + last_group = -1 + for cb in sorted(self.callbacks, key=lambda c: c.CallbackGroup): + if cb.CallbackGroup != last_group: + last_group = cb.CallbackGroup + cancel_func.write_line( + "case {0}: this->a_callback_error(({1}*)0, actor_cancelled()); break;", + cb.CallbackGroup, + cb.type, + ) + cancel_func.write_line("}") + self.WriteFunction(writer, cancel_func, cancel_func.body_text) + + def WriteConstructor( + self, body: Function, writer, full_state_class_name: str + ) -> None: + constructor = Function( + name=self.className, + return_type="", + formal_parameters=self.ParameterList(), + ) + constructor.endIsUnreachable = True + constructor.publicName = True + constructor.indent(self.codeIndent) + constructor.write_line( + " : Actor<{0}>(),".format( + "void" if self.actor.returnType is None else self.actor.returnType + ) + ) + constructor.write_line( + " {0}({1}),".format( + full_state_class_name, + ", ".join(p.name for p in self.actor.parameters), + ) + ) + constructor.write_line(" activeActorHelper(__actorIdentifier)") + constructor.indent(-1) + constructor.write_line("{") + constructor.indent(+1) + self.ProbeEnter(constructor, self.actor.name) + constructor.write_line("#ifdef ENABLE_SAMPLING") + constructor.write_line('this->lineage.setActorName("{0}");', self.actor.name) + constructor.write_line("LineageScope _(&this->lineage);") + constructor.write_line("#endif") + constructor.write_line("this->{0};", body.call()) + self.ProbeExit(constructor, self.actor.name) + self.WriteFunction(writer, constructor, constructor.body_text) + + def WriteStateConstructor(self, writer) -> None: + constructor = Function( + name=self.stateClassName, + return_type="", + formal_parameters=self.ParameterList(), + ) + constructor.endIsUnreachable = True + constructor.publicName = True + constructor.indent(self.codeIndent) + ini = None + line = self.actor.SourceLine + for state_var in self.state: + if state_var.initializer is None: + continue + self.LineNumberFunction(constructor, line) + if ini is None: + ini = " : " + else: + constructor.write_line(ini + ",") + ini = " " + ini += f"{state_var.name}({state_var.initializer})" + line = state_var.SourceLine + self.LineNumberFunction(constructor, line) + if ini: + constructor.write_line(ini) + constructor.indent(-1) + constructor.write_line("{") + constructor.indent(+1) + self.ProbeCreate(constructor, self.actor.name) + self.WriteFunction(writer, constructor, constructor.body_text) + + def WriteStateDestructor(self, writer) -> None: + destructor = Function( + name=f"~{self.stateClassName}", + return_type="", + formal_parameters=[], + ) + destructor.endIsUnreachable = True + destructor.publicName = True + destructor.indent(self.codeIndent) + destructor.indent(-1) + destructor.write_line("{") + destructor.indent(+1) + self.ProbeDestroy(destructor, self.actor.name) + self.WriteFunction(writer, destructor, destructor.body_text) + + def Compile( + self, block: CodeBlock, context: Context, ok_to_continue: bool = True + ) -> Context: + cx = context.clone() + cx.next = None + for stmt in block.statements: + if cx.target is None: + raise ActorCompilerError(stmt.FirstSourceLine, "Unreachable code.") + if cx.next is None: + cx.next = self.getFunction(cx.target.name, "cont", self.loopDepth) + self.compile_statement(stmt, cx) + if cx.next.wasCalled: + if cx.target is None: + raise ActorCompilerError( + stmt.FirstSourceLine, "Unreachable continuation called?" + ) + if not ok_to_continue: + raise ActorCompilerError( + stmt.FirstSourceLine, "Unexpected continuation" + ) + cx.target = cx.next + cx.next = None + return cx + + def compile_statement(self, stmt: Statement, cx: Context) -> None: + handler_name = f"_compile_{stmt.__class__.__name__}" + handler = getattr(self, handler_name, None) + if handler is None: + raise ActorCompilerError( + stmt.FirstSourceLine, + "Statement type {0} not supported yet.", + stmt.__class__.__name__, + ) + handler(stmt, cx) + + def _is_top_level_state_decl(self, stmt: StateDeclarationStatement) -> bool: + for candidate in self.actor.body.statements: + if not isinstance(candidate, StateDeclarationStatement): + break + if candidate is stmt: + return True + return False + + def _compile_PlainOldCodeStatement( + self, stmt: PlainOldCodeStatement, cx: Context + ) -> None: + self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + cx.target.write_line(stmt.code) + + def _compile_StateDeclarationStatement( + self, stmt: StateDeclarationStatement, cx: Context + ) -> None: + if self._is_top_level_state_decl(stmt): + self.state.append( + StateVar( + SourceLine=stmt.FirstSourceLine, + name=stmt.decl.name, + type=stmt.decl.type, + initializer=stmt.decl.initializer, + initializerConstructorSyntax=stmt.decl.initializerConstructorSyntax, + ) + ) + else: + self.state.append( + StateVar( + SourceLine=stmt.FirstSourceLine, + name=stmt.decl.name, + type=stmt.decl.type, + initializer=None, + ) + ) + if stmt.decl.initializer is not None: + self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + if ( + stmt.decl.initializerConstructorSyntax + or stmt.decl.initializer == "" + ): + cx.target.write_line( + "{0} = {1}({2});", + stmt.decl.name, + stmt.decl.type, + stmt.decl.initializer, + ) + else: + cx.target.write_line( + "{0} = {1};", stmt.decl.name, stmt.decl.initializer + ) + + def getIteratorName(self, cx: Context) -> str: + name = f"RangeFor{cx.target.name}Iterator" + self.iterators.setdefault(name, 0) + idx = self.iterators[name] + self.iterators[name] += 1 + return f"{name}{idx}" + + def EmitNativeLoop( + self, source_line: int, head: str, body: Statement, cx: Context + ) -> bool: + self.LineNumberFunction(cx.target, source_line) + cx.target.write_line(head + " {") + cx.target.indent(+1) + literal_break = LiteralBreak() + literal_continue = LiteralContinue() + self.Compile( + self.AsCodeBlock(body), + cx.loop_context(cx.target, literal_break, literal_continue, 0), + True, + ) + cx.target.indent(-1) + cx.target.write_line("}") + return not literal_break.wasCalled + + def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: + no_condition = stmt.condExpression in ("", "true", "1") + if not self.WillContinue(stmt.body): + if ( + self.EmitNativeLoop( + stmt.FirstSourceLine, + f"for({stmt.initExpression};{stmt.condExpression};{stmt.nextExpression})", + stmt.body, + cx, + ) + and no_condition + ): + cx.unreachable() + return + + init_stmt = PlainOldCodeStatement( + code=f"{stmt.initExpression};", FirstSourceLine=stmt.FirstSourceLine + ) + self._compile_PlainOldCodeStatement(init_stmt, cx) + + if no_condition: + full_body = stmt.body + else: + condition = IfStatement( + expression=f"!({stmt.condExpression})", + ifBody=BreakStatement(FirstSourceLine=stmt.FirstSourceLine), + FirstSourceLine=stmt.FirstSourceLine, + ) + full_body = CodeBlock( + statements=[condition] + list(self.AsCodeBlock(stmt.body).statements), + FirstSourceLine=stmt.FirstSourceLine, + ) + + loopF = self.getFunction(cx.target.name, "loopHead", self.loopDepth) + loopBody = self.getFunction(cx.target.name, "loopBody", self.loopDepth) + breakF = self.getFunction(cx.target.name, "break", self.loopDepth) + continueF = ( + loopF + if stmt.nextExpression == "" + else self.getFunction(cx.target.name, "continue", self.loopDepth) + ) + + loopF.write_line("int oldLoopDepth = ++loopDepth;") + loopF.write_line( + "while (loopDepth == oldLoopDepth) loopDepth = {0};", + loopBody.call("loopDepth"), + ) + + endLoop = self.Compile( + self.AsCodeBlock(full_body), + cx.loop_context(loopBody, breakF, continueF, +1), + True, + ).target + + if endLoop is not None and endLoop is not loopBody: + if stmt.nextExpression: + self._compile_PlainOldCodeStatement( + PlainOldCodeStatement( + code=f"{stmt.nextExpression};", + FirstSourceLine=stmt.FirstSourceLine, + ), + cx.with_target(endLoop), + ) + endLoop.write_line("if (loopDepth == 0) return {0};", loopF.call("0")) + + cx.target.write_line("loopDepth = {0};", loopF.call("loopDepth")) + + if continueF is not loopF and continueF.wasCalled: + self._compile_PlainOldCodeStatement( + PlainOldCodeStatement( + code=f"{stmt.nextExpression};", FirstSourceLine=stmt.FirstSourceLine + ), + cx.with_target(continueF), + ) + continueF.write_line("if (loopDepth == 0) return {0};", loopF.call("0")) + + if breakF.wasCalled: + self.TryCatch( + cx.with_target(breakF), + cx.catchFErr, + cx.tryLoopDepth, + lambda: breakF.write_line("return {0};", cx.next.call("loopDepth")), + ) + else: + cx.unreachable() + + def _compile_RangeForStatement(self, stmt: RangeForStatement, cx: Context) -> None: + if self.WillContinue(stmt.body): + container = next( + (s for s in self.state if s.name == stmt.rangeExpression), None + ) + if container is None: + raise ActorCompilerError( + stmt.FirstSourceLine, + "container of range-based for with continuation must be a state variable", + ) + iterator_name = self.getIteratorName(cx) + self.state.append( + StateVar( + SourceLine=stmt.FirstSourceLine, + name=iterator_name, + type=f"decltype(std::begin(std::declval<{container.type}>()))", + initializer=None, + ) + ) + equivalent = ForStatement( + initExpression=f"{iterator_name} = std::begin({stmt.rangeExpression})", + condExpression=f"{iterator_name} != std::end({stmt.rangeExpression})", + nextExpression=f"++{iterator_name}", + FirstSourceLine=stmt.FirstSourceLine, + body=CodeBlock( + statements=[ + PlainOldCodeStatement( + code=f"{stmt.rangeDecl} = *{iterator_name};", + FirstSourceLine=stmt.FirstSourceLine, + ), + stmt.body, + ], + FirstSourceLine=stmt.FirstSourceLine, + ), + ) + self._compile_ForStatement(equivalent, cx) + else: + self.EmitNativeLoop( + stmt.FirstSourceLine, + f"for( {stmt.rangeDecl} : {stmt.rangeExpression} )", + stmt.body, + cx, + ) + + def _compile_WhileStatement(self, stmt: WhileStatement, cx: Context) -> None: + equivalent = ForStatement( + initExpression="", + condExpression=stmt.expression, + nextExpression="", + body=stmt.body, + FirstSourceLine=stmt.FirstSourceLine, + ) + self._compile_ForStatement(equivalent, cx) + + def _compile_LoopStatement(self, stmt: LoopStatement, cx: Context) -> None: + equivalent = ForStatement( + initExpression="", + condExpression="", + nextExpression="", + body=stmt.body, + FirstSourceLine=stmt.FirstSourceLine, + ) + self._compile_ForStatement(equivalent, cx) + + def _compile_BreakStatement(self, stmt: BreakStatement, cx: Context) -> None: + if cx.breakF is None: + raise ActorCompilerError(stmt.FirstSourceLine, "break outside loop") + if isinstance(cx.breakF, LiteralBreak): + cx.target.write_line("{0};", cx.breakF.call()) + else: + cx.target.write_line( + "return {0}; // break", cx.breakF.call("loopDepth==0?0:loopDepth-1") + ) + cx.unreachable() + + def _compile_ContinueStatement(self, stmt: ContinueStatement, cx: Context) -> None: + if cx.continueF is None: + raise ActorCompilerError(stmt.FirstSourceLine, "continue outside loop") + if isinstance(cx.continueF, LiteralContinue): + cx.target.write_line("{0};", cx.continueF.call()) + else: + cx.target.write_line( + "return {0}; // continue", cx.continueF.call("loopDepth") + ) + cx.unreachable() + + def _compile_CodeBlock(self, stmt: CodeBlock, cx: Context) -> None: + cx.target.write_line("{") + cx.target.indent(+1) + end = self.Compile(stmt, cx, True) + cx.target.indent(-1) + cx.target.write_line("}") + if end.target is None: + cx.unreachable() + elif end.target is not cx.target: + end.target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + + def _compile_ReturnStatement(self, stmt: ReturnStatement, cx: Context) -> None: + self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + if (stmt.expression == "") != (self.actor.returnType is None): + raise ActorCompilerError( + stmt.FirstSourceLine, + "Return statement does not match actor declaration", + ) + if self.actor.returnType is not None: + if stmt.expression == "Never()": + cx.target.write_line("this->~{0}();", self.stateClassName) + cx.target.write_line("{0}->sendAndDelPromiseRef(Never());", self.This) + else: + cx.target.write_line( + "if (!{0}->SAV<{1}>::futures) {{ (void)({2}); this->~{3}(); {0}->destroy(); return 0; }}", + self.This, + self.actor.returnType, + stmt.expression, + self.stateClassName, + ) + if any(s.name == stmt.expression for s in self.state): + cx.target.write_line( + "new (&{0}->SAV< {1} >::value()) {1}(std::move({2})); // state_var_RVO", + self.This, + self.actor.returnType, + stmt.expression, + ) + else: + cx.target.write_line( + "new (&{0}->SAV< {1} >::value()) {1}({2});", + self.This, + self.actor.returnType, + stmt.expression, + ) + cx.target.write_line("this->~{0}();", self.stateClassName) + cx.target.write_line("{0}->finishSendAndDelPromiseRef();", self.This) + else: + cx.target.write_line("delete {0};", self.This) + cx.target.write_line("return 0;") + cx.unreachable() + + def _compile_ThrowStatement(self, stmt: ThrowStatement, cx: Context) -> None: + self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + if stmt.expression == "": + if cx.target.exceptionParameterIs is not None: + cx.target.write_line( + "return {0};", + cx.catchFErr.call( + cx.target.exceptionParameterIs, + self.AdjustLoopDepth(cx.tryLoopDepth), + ), + ) + else: + raise ActorCompilerError( + stmt.FirstSourceLine, + "throw statement with no expression has no current exception in scope", + ) + else: + cx.target.write_line( + "return {0};", + cx.catchFErr.call( + stmt.expression, self.AdjustLoopDepth(cx.tryLoopDepth) + ), + ) + cx.unreachable() + + def _compile_IfStatement(self, stmt: IfStatement, cx: Context) -> None: + use_continuation = self.WillContinue(stmt.ifBody) or ( + stmt.elseBody is not None and self.WillContinue(stmt.elseBody) + ) + self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + constexpr = "constexpr " if stmt.constexpr else "" + cx.target.write_line(f"if {constexpr}({stmt.expression})") + cx.target.write_line("{") + cx.target.indent(+1) + if_target = self.Compile( + self.AsCodeBlock(stmt.ifBody), cx, use_continuation + ).target + if use_continuation and if_target is not None: + if_target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + cx.target.indent(-1) + cx.target.write_line("}") + else_target = None + if stmt.elseBody is not None or use_continuation: + cx.target.write_line("else") + cx.target.write_line("{") + cx.target.indent(+1) + else_target = cx.target + if stmt.elseBody is not None: + else_target = self.Compile( + self.AsCodeBlock(stmt.elseBody), cx, use_continuation + ).target + if use_continuation and else_target is not None: + else_target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + cx.target.indent(-1) + cx.target.write_line("}") + if if_target is None and stmt.elseBody is not None and else_target is None: + cx.unreachable() + elif not cx.next.wasCalled and use_continuation: + raise ActorCompilerError( + stmt.FirstSourceLine, "Internal error: IfStatement: next not called?" + ) + + def _compile_TryStatement(self, stmt: TryStatement, cx: Context) -> None: + if len(stmt.catches) != 1: + raise ActorCompilerError( + stmt.FirstSourceLine, + "try statement must have exactly one catch clause", + ) + reachable = False + catch_clause = stmt.catches[0] + catch_expression = catch_clause.expression.replace(" ", "") + catch_param = "" + if catch_expression != "...": + if not catch_expression.startswith("Error&"): + raise ActorCompilerError( + catch_clause.FirstSourceLine, + "Only type 'Error' or '...' may be caught in an actor function", + ) + catch_param = catch_expression[6:] + if not catch_param: + catch_param = "__current_error" + catchFErr = self.getFunction( + cx.target.name, "Catch", f"const Error& {catch_param}", self.loopDepth0 + ) + catchFErr.exceptionParameterIs = catch_param + end = self.TryCatchCompile( + self.AsCodeBlock(stmt.tryBody), cx.with_catch(catchFErr) + ) + if end.target is not None: + reachable = True + self.TryCatch( + end, + cx.catchFErr, + cx.tryLoopDepth, + lambda: end.target.write_line( + "loopDepth = {0};", cx.next.call("loopDepth") + ), + ) + + def handle_catch() -> None: + nonlocal reachable + cend = self._compile_try_catch_body(catch_clause, catchFErr, cx) + if cend.target is not None: + reachable = True + + self.TryCatch( + cx.with_target(catchFErr), + cx.catchFErr, + cx.tryLoopDepth, + handle_catch, + ) + if not reachable: + cx.unreachable() + + def _compile_try_catch_body( + self, catch_clause: TryStatement.Catch, catchFErr: Function, cx: Context + ) -> Context: + cend = self.Compile( + self.AsCodeBlock(catch_clause.body), cx.with_target(catchFErr), True + ) + if cend.target is not None: + cend.target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + return cend + + def _compile_WaitStatement(self, stmt: WaitStatement, cx: Context) -> None: + equivalent = ChooseStatement( + body=CodeBlock( + statements=[ + WhenStatement( + wait=stmt, + body=None, + FirstSourceLine=stmt.FirstSourceLine, + ) + ], + FirstSourceLine=stmt.FirstSourceLine, + ), + FirstSourceLine=stmt.FirstSourceLine, + ) + if not stmt.resultIsState: + cx.next.formalParameters = [ + f"{stmt.result.type} const& {stmt.result.name}", + self.loopDepth, + ] + cx.next.add_overload( + f"{stmt.result.type} && {stmt.result.name}", self.loopDepth + ) + self._compile_ChooseStatement(equivalent, cx) + + def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None: + group = self.chooseGroups + 1 + self.chooseGroups = group + codeblock = stmt.body if isinstance(stmt.body, CodeBlock) else None + if codeblock is None: + raise ActorCompilerError( + stmt.FirstSourceLine, + "'choose' must be followed by a compound statement.", + ) + choices = [] + for idx, choice_stmt in enumerate(codeblock.statements): + if not isinstance(choice_stmt, WhenStatement): + raise ActorCompilerError( + choice_stmt.FirstSourceLine, + "only 'when' statements are valid in a 'choose' block.", + ) + index = self.whenCount + idx + param_prefix = "__" if choice_stmt.wait.resultIsState else "" + const_param = f"{choice_stmt.wait.result.type} const& {param_prefix}{choice_stmt.wait.result.name}" + rvalue_param = f"{choice_stmt.wait.result.type} && {param_prefix}{choice_stmt.wait.result.name}" + body_func = self.getFunction( + cx.target.name, + "when", + [const_param, self.loopDepth], + overload_formal_parameters=[rvalue_param, self.loopDepth], + ) + future_name = f"__when_expr_{index}" + callback_template = ( + "ActorSingleCallback" + if choice_stmt.wait.isWaitNext + else "ActorCallback" + ) + callback_type = f"{callback_template}< {self.fullClassName}, {index}, {choice_stmt.wait.result.type} >" + callback_type_state = f"{callback_template}< {self.className}, {index}, {choice_stmt.wait.result.type} >" + choices.append( + { + "Stmt": choice_stmt, + "Group": group, + "Index": index, + "Body": body_func, + "Future": future_name, + "CallbackType": callback_type, + "CallbackTypeState": callback_type_state, + } + ) + self.whenCount += len(choices) + exit_func = self.getFunction("exitChoose", "", []) + exit_func.returnType = "void" + exit_func.write_line( + "if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", self.This + ) + for choice in choices: + exit_func.write_line( + "{0}->{1}::remove();", self.This, choice["CallbackTypeState"] + ) + exit_func.endIsUnreachable = True + + reachable = False + for choice in choices: + self.callbacks.append( + CallbackVar( + SourceLine=choice["Stmt"].FirstSourceLine, + CallbackGroup=choice["Group"], + type=choice["CallbackType"], + ) + ) + r = choice["Body"] + if choice["Stmt"].wait.resultIsState: + overload = r.pop_overload() + decl_stmt = StateDeclarationStatement( + decl=VarDeclaration( + type=choice["Stmt"].wait.result.type, + name=choice["Stmt"].wait.result.name, + initializer=f"__{choice['Stmt'].wait.result.name}", + initializerConstructorSyntax=False, + ), + FirstSourceLine=choice["Stmt"].FirstSourceLine, + ) + self._compile_StateDeclarationStatement(decl_stmt, cx.with_target(r)) + if overload is not None: + overload.write_line( + "{0} = std::move(__{0});", + choice["Stmt"].wait.result.name, + ) + r.set_overload(overload) + if choice["Stmt"].body is not None: + r = self.Compile( + self.AsCodeBlock(choice["Stmt"].body), cx.with_target(r), True + ).target + if r is not None: + reachable = True + if len(cx.next.formalParameters) == 1: + r.write_line("loopDepth = {0};", cx.next.call("loopDepth")) + else: + overload = r.pop_overload() + r.write_line( + "loopDepth = {0};", + cx.next.call(choice["Stmt"].wait.result.name, "loopDepth"), + ) + if overload is not None: + overload.write_line( + "loopDepth = {0};", + cx.next.call( + f"std::move({choice['Stmt'].wait.result.name})", + "loopDepth", + ), + ) + r.set_overload(overload) + + cb_func = Function( + name="callback_fire", + return_type="void", + formal_parameters=[ + f"{choice['CallbackTypeState']}*", + f"{choice['Stmt'].wait.result.type} const& value", + ], + ) + cb_func.endIsUnreachable = True + cb_func.add_overload( + f"{choice['CallbackTypeState']}*", + f"{choice['Stmt'].wait.result.type} && value", + ) + cb_func.indent(self.codeIndent) + self.ProbeEnter(cb_func, self.actor.name, choice["Index"]) + cb_func.write_line("{0};", exit_func.call()) + overload_func = cb_func.pop_overload() + + def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None: + self.TryCatch( + cx.with_target(target_func), + cx.catchFErr, + cx.tryLoopDepth, + lambda ch=choice_dict, tf=target_func, expression=expr: tf.write_line( + "{0};", ch["Body"].call(expression, "0") + ), + False, + ) + + fire_body(cb_func, "value") + if overload_func is not None: + fire_body(overload_func, "std::move(value)") + cb_func.set_overload(overload_func) + self.ProbeExit(cb_func, self.actor.name, choice["Index"]) + self.functions[f"{cb_func.name}#{choice['Index']}"] = cb_func + + err_func = Function( + name="callback_error", + return_type="void", + formal_parameters=[f"{choice['CallbackTypeState']}*", "Error err"], + ) + err_func.endIsUnreachable = True + err_func.indent(self.codeIndent) + self.ProbeEnter(err_func, self.actor.name, choice["Index"]) + err_func.write_line("{0};", exit_func.call()) + self.TryCatch( + cx.with_target(err_func), + cx.catchFErr, + cx.tryLoopDepth, + lambda: err_func.write_line("{0};", cx.catchFErr.call("err", "0")), + False, + ) + self.ProbeExit(err_func, self.actor.name, choice["Index"]) + self.functions[f"{err_func.name}#{choice['Index']}"] = err_func + + first_choice = True + for choice in choices: + get_func = "pop" if choice["Stmt"].wait.isWaitNext else "get" + self.LineNumberFunction(cx.target, choice["Stmt"].wait.FirstSourceLine) + if choice["Stmt"].wait.isWaitNext: + cx.target.write_line( + "auto {0} = {1};", + choice["Future"], + choice["Stmt"].wait.futureExpression, + ) + cx.target.write_line( + 'static_assert(std::is_same>::value || std::is_same>::value, "invalid type");', + choice["Future"], + choice["Stmt"].wait.result.type, + ) + else: + cx.target.write_line( + "StrictFuture<{0}> {1} = {2};", + choice["Stmt"].wait.result.type, + choice["Future"], + choice["Stmt"].wait.futureExpression, + ) + if first_choice: + first_choice = False + self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + if self.actor.IsCancellable(): + cx.target.write_line( + "if ({1}->actor_wait_state < 0) return {0};", + cx.catchFErr.call( + "actor_cancelled()", self.AdjustLoopDepth(cx.tryLoopDepth) + ), + self.This, + ) + cx.target.write_line( + "if ({0}.isReady()) {{ if ({0}.isError()) return {2}; else return {1}; }};", + choice["Future"], + choice["Body"].call(f"{choice['Future']}.{get_func}()", "loopDepth"), + cx.catchFErr.call( + f"{choice['Future']}.getError()", + self.AdjustLoopDepth(cx.tryLoopDepth), + ), + ) + cx.target.write_line("{1}->actor_wait_state = {0};", group, self.This) + for choice in choices: + self.LineNumberFunction(cx.target, choice["Stmt"].wait.FirstSourceLine) + cx.target.write_line( + "{0}.addCallbackAndClear(static_cast<{1}*>({2}));", + choice["Future"], + choice["CallbackTypeState"], + self.This, + ) + cx.target.write_line("loopDepth = 0;") + if not reachable: + cx.unreachable() diff --git a/flow/actorcompiler/actor_parser.py b/flow/actorcompiler/actor_parser.py new file mode 100644 index 00000000000..f45977eeaa9 --- /dev/null +++ b/flow/actorcompiler/actor_parser.py @@ -0,0 +1,1134 @@ +from __future__ import annotations + +import io +import re +import sys +from dataclasses import dataclass +from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple + +from . import ActorCompilerError +from .actor_compiler import ActorCompiler, DescrCompiler + +from .parse_tree import ( + Actor, + BreakStatement, + ChooseStatement, + CodeBlock, + ContinueStatement, + Declaration, + Descr, + ForStatement, + IfStatement, + LoopStatement, + PlainOldCodeStatement, + RangeForStatement, + ReturnStatement, + StateDeclarationStatement, + Statement, + ThrowStatement, + TryStatement, + VarDeclaration, + WaitStatement, + WhenStatement, + WhileStatement, +) + + +class ErrorMessagePolicy: + def __init__(self) -> None: + self.DisableDiagnostics = False + + def HandleActorWithoutWait(self, sourceFile: str, actor: Actor) -> None: + if not self.DisableDiagnostics and not actor.isTestCase: + print( + f"{sourceFile}:{actor.SourceLine}: warning: ACTOR {actor.name} does not contain a wait() statement", + file=sys.stderr, + ) + + def ActorsNoDiscardByDefault(self) -> bool: + return not self.DisableDiagnostics + + +@dataclass +class Token: + Value: str + Position: int = 0 + SourceLine: int = 0 + BraceDepth: int = 0 + ParenDepth: int = 0 + + @property + def IsWhitespace(self) -> bool: + return ( + self.Value in (" ", "\n", "\r", "\r\n", "\t") + or self.Value.startswith("//") + or self.Value.startswith("/*") + ) + + def Assert(self, error: str, pred) -> "Token": + if not pred(self): + raise ActorCompilerError(self.SourceLine, error) + return self + + def GetMatchingRangeIn(self, token_range: "TokenRange") -> "TokenRange": + if self.Value == "<": + sub = TokenRange(token_range.tokens, self.Position, token_range.End) + gen = AngleBracketParser.NotInsideAngleBrackets(sub) + next(gen, None) # skip the "<" + closing = next(gen, None) + if closing is None: + raise ActorCompilerError(self.SourceLine, "Syntax error: Unmatched <") + return TokenRange(token_range.tokens, self.Position + 1, closing.Position) + if self.Value == "[": + sub = TokenRange(token_range.tokens, self.Position, token_range.End) + gen = BracketParser.NotInsideBrackets(sub) + next(gen, None) # skip the "[" + closing = next(gen, None) + if closing is None: + raise ActorCompilerError(self.SourceLine, "Syntax error: Unmatched [") + return TokenRange(token_range.tokens, self.Position + 1, closing.Position) + + pairs = {"(": ")", ")": "(", "{": "}", "}": "{"} + if self.Value not in pairs: + raise RuntimeError("Can't match this token") + pred = ( + (lambda t: t.Value != ")" or t.ParenDepth != self.ParenDepth) + if self.Value == "(" + else ( + (lambda t: t.Value != "(" or t.ParenDepth != self.ParenDepth) + if self.Value == ")" + else ( + (lambda t: t.Value != "}" or t.BraceDepth != self.BraceDepth) + if self.Value == "{" + else (lambda t: t.Value != "{" or t.BraceDepth != self.BraceDepth) + ) + ) + ) + direction = 1 if self.Value in ("(", "{") else -1 + if direction == -1: + rng = token_range.Range(token_range.Begin, self.Position).RevTakeWhile(pred) + if rng.Begin == token_range.Begin: + raise ActorCompilerError( + self.SourceLine, f"Syntax error: Unmatched {self.Value}" + ) + return rng + rng = token_range.Range(self.Position + 1, token_range.End).TakeWhile(pred) + if rng.End == token_range.End: + raise ActorCompilerError( + self.SourceLine, f"Syntax error: Unmatched {self.Value}" + ) + return rng + + +class TokenRange: + def __init__(self, tokens: List[Token], begin: int, end: int) -> None: + if begin > end: + raise RuntimeError("Invalid TokenRange") + self.tokens = tokens + self.beginPos = begin + self.endPos = end + + @property + def Begin(self) -> int: + return self.beginPos + + @property + def End(self) -> int: + return self.endPos + + def __iter__(self) -> Iterator[Token]: + for i in range(self.beginPos, self.endPos): + yield self.tokens[i] + + def First(self, predicate=None) -> Token: + if self.beginPos == self.endPos: + raise RuntimeError("Empty TokenRange") + if predicate is None: + return self.tokens[self.beginPos] + for t in self: + if predicate(t): + return t + raise RuntimeError("Matching token not found") + + def Last(self, predicate=None) -> Token: + if self.beginPos == self.endPos: + raise RuntimeError("Empty TokenRange") + if predicate is None: + return self.tokens[self.endPos - 1] + for i in range(self.endPos - 1, self.beginPos - 1, -1): + if predicate(self.tokens[i]): + return self.tokens[i] + raise RuntimeError("Matching token not found") + + def Skip(self, count: int) -> "TokenRange": + return TokenRange(self.tokens, self.beginPos + count, self.endPos) + + def Consume(self, value_or_error, predicate=None) -> "TokenRange": + if predicate is None: + self.First().Assert( + f"Expected {value_or_error}", lambda t: t.Value == value_or_error + ) + else: + self.First().Assert(value_or_error, predicate) + return self.Skip(1) + + def SkipWhile(self, predicate) -> "TokenRange": + e = self.beginPos + while e < self.endPos and predicate(self.tokens[e]): + e += 1 + return TokenRange(self.tokens, e, self.endPos) + + def TakeWhile(self, predicate) -> "TokenRange": + e = self.beginPos + while e < self.endPos and predicate(self.tokens[e]): + e += 1 + return TokenRange(self.tokens, self.beginPos, e) + + def RevTakeWhile(self, predicate) -> "TokenRange": + e = self.endPos - 1 + while e >= self.beginPos and predicate(self.tokens[e]): + e -= 1 + return TokenRange(self.tokens, e + 1, self.endPos) + + def RevSkipWhile(self, predicate) -> "TokenRange": + e = self.endPos - 1 + while e >= self.beginPos and predicate(self.tokens[e]): + e -= 1 + return TokenRange(self.tokens, self.beginPos, e + 1) + + def Range(self, begin: int, end: int) -> "TokenRange": + return TokenRange(self.tokens, begin, end) + + def IsEmpty(self) -> bool: + return self.beginPos == self.endPos + + def Length(self) -> int: + return self.endPos - self.beginPos + + def All(self, predicate) -> bool: + return all(predicate(t) for t in self) + + def Any(self, predicate) -> bool: + return any(predicate(t) for t in self) + + +class BracketParser: + @staticmethod + def NotInsideBrackets(tokens: Iterable[Token]) -> Iterator[Token]: + bracket_depth = 0 + base_pd = None + for tok in tokens: + if base_pd is None: + base_pd = tok.ParenDepth + if tok.ParenDepth == base_pd and tok.Value == "]": + bracket_depth -= 1 + if bracket_depth == 0: + yield tok + if tok.ParenDepth == base_pd and tok.Value == "[": + bracket_depth += 1 + + +class AngleBracketParser: + @staticmethod + def NotInsideAngleBrackets(tokens: Iterable[Token]) -> Iterator[Token]: + angle_depth = 0 + base_pd = None + for tok in tokens: + if base_pd is None: + base_pd = tok.ParenDepth + if tok.ParenDepth == base_pd and tok.Value == ">": + angle_depth -= 1 + if angle_depth == 0: + yield tok + if tok.ParenDepth == base_pd and tok.Value == "<": + angle_depth += 1 + + +class ActorParser: + tokenExpressions = [ + r"\{", + r"\}", + r"\(", + r"\)", + r"\[", + r"\]", + r"//[^\n]*", + r"/[*]([*][^/]|[^*])*[*]/", + r"'(\\.|[^\'\n])*'", + r'"(\\.|[^"\n])*"', + r"[a-zA-Z_][a-zA-Z_0-9]*", + r"\r\n", + r"\n", + r"::", + r":", + r"#[a-z]*", + r".", + ] + + identifierPattern = re.compile(r"^[a-zA-Z_][a-zA-Z_0-9]*$") + + def __init__( + self, + text: str, + sourceFile: str, + errorMessagePolicy: ErrorMessagePolicy, + generateProbes: bool, + ) -> None: + self.sourceFile = sourceFile + self.errorMessagePolicy = errorMessagePolicy + self.generateProbes = generateProbes + self.tokens = [Token(Value=t) for t in self.Tokenize(text)] + self.LineNumbersEnabled = True + self.uidObjects: Dict[Tuple[int, int], str] = {} + self.TokenArray = self.tokens + self.CountParens() + + def Tokenize(self, text: str) -> List[str]: + regexes = [re.compile(pattern, re.S) for pattern in self.tokenExpressions] + pos = 0 + tokens: List[str] = [] + while pos < len(text): + for regex in regexes: + m = regex.match(text, pos) + if m: + tokens.append(m.group(0)) + pos += len(m.group(0)) + break + else: + raise RuntimeError(f"Can't tokenize! {pos}") + return tokens + + def CountParens(self) -> None: + brace_depth = 0 + paren_depth = 0 + line_count = 1 + last_paren = None + last_brace = None + for i, token in enumerate(self.tokens): + value = token.Value + if value == "}": + brace_depth -= 1 + elif value == ")": + paren_depth -= 1 + elif value == "\r\n": + line_count += 1 + elif value == "\n": + line_count += 1 + + if brace_depth < 0: + raise ActorCompilerError(line_count, "Mismatched braces") + if paren_depth < 0: + raise ActorCompilerError(line_count, "Mismatched parenthesis") + + token.Position = i + token.SourceLine = line_count + token.BraceDepth = brace_depth + token.ParenDepth = paren_depth + + if value.startswith("/*"): + line_count += value.count("\n") + if value == "{": + brace_depth += 1 + if brace_depth == 1: + last_brace = token + elif value == "(": + paren_depth += 1 + if paren_depth == 1: + last_paren = token + if brace_depth != 0: + raise ActorCompilerError( + last_brace.SourceLine if last_brace else line_count, "Unmatched brace" + ) + if paren_depth != 0: + raise ActorCompilerError( + last_paren.SourceLine if last_paren else line_count, + "Unmatched parenthesis", + ) + + def Write(self, writer: io.TextIOBase, destFileName: str) -> None: + ActorCompiler.usedClassNames.clear() + writer.write("#define POST_ACTOR_COMPILER 1\n") + outLine = 1 + if self.LineNumbersEnabled: + writer.write(f'#line {self.tokens[0].SourceLine} "{self.sourceFile}"\n') + outLine += 1 + inBlocks = 0 + classContextStack: List[Tuple[str, int]] = [] + i = 0 + while i < len(self.tokens): + tok = self.tokens[i] + if tok.SourceLine == 0: + raise RuntimeError("Invalid source line (0)") + if tok.Value in ("ACTOR", "SWIFT_ACTOR", "TEST_CASE"): + actor = self.ParseActor(i) + end = self._parse_end + if classContextStack: + actor.enclosingClass = "::".join( + name for name, _ in classContextStack + ) + actor_writer = io.StringIO() + actorCompiler = ActorCompiler( + actor, + self.sourceFile, + inBlocks == 0, + self.LineNumbersEnabled, + self.generateProbes, + ) + actorCompiler.Write(actor_writer) + for key, value in actorCompiler.uidObjects.items(): + self.uidObjects.setdefault(key, value) + actor_lines = actor_writer.getvalue().split("\n") + hasLineNumber = False + hadLineNumber = True + for line in actor_lines: + if self.LineNumbersEnabled: + isLine = "#line" in line + if isLine: + hadLineNumber = True + if not isLine and not hasLineNumber and hadLineNumber: + writer.write( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( + outLine + 1, destFileName + ) + ) + outLine += 1 + hadLineNumber = False + hasLineNumber = isLine + writer.write(line.rstrip("\n\r") + "\n") + outLine += 1 + i = end + if i < len(self.tokens) and self.LineNumbersEnabled: + writer.write( + f'#line {self.tokens[i].SourceLine} "{self.sourceFile}"\n' + ) + outLine += 1 + elif tok.Value == "DESCR": + descr, end = self.ParseDescr(i) + descr_writer = io.StringIO() + lines = DescrCompiler(descr, tok.BraceDepth).write(descr_writer) + writer.write(descr_writer.getvalue()) + outLine += lines + i = end + if i < len(self.tokens) and self.LineNumbersEnabled: + writer.write( + f'#line {self.tokens[i].SourceLine} "{self.sourceFile}"\n' + ) + outLine += 1 + elif tok.Value in ("class", "struct", "union"): + writer.write(tok.Value) + success, name = self.ParseClassContext( + self.range(i + 1, len(self.tokens)) + ) + if success: + classContextStack.append((name, inBlocks)) + else: + if tok.Value == "{": + inBlocks += 1 + elif tok.Value == "}": + inBlocks -= 1 + if classContextStack and classContextStack[-1][1] == inBlocks: + classContextStack.pop() + writer.write(tok.Value) + outLine += tok.Value.count("\n") + i += 1 + + # Parsing helpers + + def range(self, begin: int, end: int) -> TokenRange: + return TokenRange(self.tokens, begin, end) + + def ParseActor(self, pos: int) -> Actor: + token = self.tokens[pos] + actor = Actor() + actor.SourceLine = token.SourceLine + toks = self.range(pos + 1, len(self.tokens)) + heading = toks.TakeWhile(lambda t: t.Value != "{") + toSemicolon = toks.TakeWhile(lambda t: t.Value != ";") + actor.isForwardDeclaration = toSemicolon.Length() < heading.Length() + if actor.isForwardDeclaration: + heading = toSemicolon + if token.Value in ("ACTOR", "SWIFT_ACTOR"): + self.ParseActorHeading(actor, heading) + else: + token.Assert("ACTOR expected!", lambda _: False) + self._parse_end = heading.End + 1 + else: + body = self.range(heading.End + 1, len(self.tokens)).TakeWhile( + lambda t: t.BraceDepth > toks.First().BraceDepth + ) + if token.Value in ("ACTOR", "SWIFT_ACTOR"): + self.ParseActorHeading(actor, heading) + elif token.Value == "TEST_CASE": + self.ParseTestCaseHeading(actor, heading) + actor.isTestCase = True + else: + token.Assert("ACTOR or TEST_CASE expected!", lambda _: False) + actor.body = self.ParseCodeBlock(body) + if not actor.body.containsWait(): + self.errorMessagePolicy.HandleActorWithoutWait(self.sourceFile, actor) + self._parse_end = body.End + 1 + return actor + + def ParseDescr(self, pos: int) -> Tuple[Descr, int]: + descr = Descr() + toks = self.range(pos + 1, len(self.tokens)) + heading = toks.TakeWhile(lambda t: t.Value != "{") + body = self.range(heading.End + 1, len(self.tokens)).TakeWhile( + lambda t: t.BraceDepth > toks.First().BraceDepth or t.Value == ";" + ) + self.ParseDescrHeading(descr, heading) + descr.body = self.ParseDescrCodeBlock(body) + end = body.End + 1 + return descr, end + + def ParseDescrHeading(self, descr: Descr, toks: TokenRange) -> None: + nonWhitespace = lambda t: not t.IsWhitespace + toks.First(nonWhitespace).Assert( + "non-struct DESCR!", lambda t: t.Value == "struct" + ) + toks = ( + toks.SkipWhile(lambda t: t.IsWhitespace) + .Skip(1) + .SkipWhile(lambda t: t.IsWhitespace) + ) + colon = next((t for t in toks if t.Value == ":"), None) + if colon: + descr.superClassList = self.str( + self.range(colon.Position + 1, toks.End) + ).strip() + toks = self.range(toks.Begin, colon.Position) + descr.name = self.str(toks).strip() + + def ParseDescrCodeBlock(self, toks: TokenRange) -> List[Declaration]: + declarations: List[Declaration] = [] + while True: + delim = next((t for t in toks if t.Value == ";"), None) + if delim is None: + break + pos = delim.Position + 1 + potential_comment = self.range(pos, toks.End).SkipWhile( + lambda t: t.Value in ("\t", " ") + ) + if ( + not potential_comment.IsEmpty() + and potential_comment.First().Value.startswith("//") + ): + pos = potential_comment.First().Position + 1 + self.ParseDeclarationRange(self.range(toks.Begin, pos), declarations) + toks = self.range(pos, toks.End) + if not toks.All(lambda t: t.IsWhitespace): + raise ActorCompilerError( + toks.First(lambda t: not t.IsWhitespace).SourceLine, + "Trailing unterminated statement in code block", + ) + return declarations + + def ParseDeclarationRange( + self, toks: TokenRange, declarations: List[Declaration] + ) -> None: + delim = toks.First(lambda t: t.Value == ";") + nameRange = ( + self.range(toks.Begin, delim.Position) + .RevSkipWhile(lambda t: t.IsWhitespace) + .RevTakeWhile(lambda t: not t.IsWhitespace) + ) + typeRange = self.range(toks.Begin, nameRange.Begin) + commentRange = self.range(delim.Position + 1, toks.End) + declarations.append( + Declaration( + name=self.str(nameRange).strip(), + type=self.str(typeRange).strip(), + comment=self.str(commentRange).strip().lstrip("/"), + ) + ) + + def ParseTestCaseHeading(self, actor: Actor, toks: TokenRange) -> None: + actor.isStatic = True + nonWhitespace = lambda t: not t.IsWhitespace + paramRange = ( + toks.Last(nonWhitespace) + .Assert( + "Unexpected tokens after test case parameter list.", + lambda t: t.Value == ")" and t.ParenDepth == toks.First().ParenDepth, + ) + .GetMatchingRangeIn(toks) + ) + actor.testCaseParameters = self.str(paramRange) + actor.name = f"flowTestCase{toks.First().SourceLine}" + actor.parameters = [ + VarDeclaration( + name="params", + type="UnitTestParameters", + initializer="", + initializerConstructorSyntax=False, + ) + ] + actor.returnType = "Void" + + def ParseActorHeading(self, actor: Actor, toks: TokenRange) -> None: + nonWhitespace = lambda t: not t.IsWhitespace + template = toks.First(nonWhitespace) + if template.Value == "template": + templateParams = ( + self.range(template.Position + 1, toks.End) + .First(nonWhitespace) + .Assert("Invalid template declaration", lambda t: t.Value == "<") + .GetMatchingRangeIn(toks) + ) + actor.templateFormals = [ + self.ParseVarDeclaration(p) + for p in self.SplitParameterList(templateParams, ",") + ] + toks = self.range(templateParams.End + 1, toks.End) + attribute = toks.First(nonWhitespace) + while attribute.Value == "[": + contents = attribute.GetMatchingRangeIn(toks) + as_array = list(contents) + if ( + len(as_array) < 2 + or as_array[0].Value != "[" + or as_array[-1].Value != "]" + ): + raise ActorCompilerError( + actor.SourceLine, "Invalid attribute: Expected [[...]]" + ) + actor.attributes.append( + "[" + self.str(self.NormalizeWhitespace(contents)) + "]" + ) + toks = self.range(contents.End + 1, toks.End) + attribute = toks.First(nonWhitespace) + static_keyword = toks.First(nonWhitespace) + if static_keyword.Value == "static": + actor.isStatic = True + toks = self.range(static_keyword.Position + 1, toks.End) + uncancellable = toks.First(nonWhitespace) + if uncancellable.Value == "UNCANCELLABLE": + actor.SetUncancellable() + toks = self.range(uncancellable.Position + 1, toks.End) + paramRange = ( + toks.Last(nonWhitespace) + .Assert( + "Unexpected tokens after actor parameter list.", + lambda t: t.Value == ")" and t.ParenDepth == toks.First().ParenDepth, + ) + .GetMatchingRangeIn(toks) + ) + actor.parameters = [ + self.ParseVarDeclaration(p) + for p in self.SplitParameterList(paramRange, ",") + ] + nameToken = self.range(toks.Begin, paramRange.Begin - 1).Last(nonWhitespace) + actor.name = nameToken.Value + return_range = self.range( + toks.First().Position + 1, nameToken.Position + ).SkipWhile(lambda t: t.IsWhitespace) + retToken = return_range.First() + if retToken.Value == "Future": + ofType = ( + return_range.Skip(1) + .First(nonWhitespace) + .Assert("Expected <", lambda tok: tok.Value == "<") + .GetMatchingRangeIn(return_range) + ) + actor.returnType = self.str(self.NormalizeWhitespace(ofType)) + toks = self.range(ofType.End + 1, return_range.End) + elif retToken.Value == "void": + actor.returnType = None + toks = return_range.Skip(1) + else: + raise ActorCompilerError( + actor.SourceLine, "Actor apparently does not return Future" + ) + toks = toks.SkipWhile(lambda t: t.IsWhitespace) + if not toks.IsEmpty(): + if toks.Last().Value == "::": + actor.nameSpace = self.str(self.range(toks.Begin, toks.End - 1)) + else: + raise ActorCompilerError( + actor.SourceLine, + "Unrecognized tokens preceding parameter list in actor declaration", + ) + if ( + self.errorMessagePolicy.ActorsNoDiscardByDefault() + and "[[flow_allow_discard]]" not in actor.attributes + ): + if actor.IsCancellable(): + actor.attributes.append("[[nodiscard]]") + known_flow_attributes = {"[[flow_allow_discard]]"} + for flow_attribute in [a for a in actor.attributes if a.startswith("[[flow_")]: + if flow_attribute not in known_flow_attributes: + raise ActorCompilerError( + actor.SourceLine, f"Unknown flow attribute {flow_attribute}" + ) + actor.attributes = [a for a in actor.attributes if not a.startswith("[[flow_")] + + def ParseVarDeclaration(self, tokens: TokenRange) -> VarDeclaration: + name, typeRange, initializer, constructorSyntax = self.ParseDeclaration(tokens) + return VarDeclaration( + name=name.Value, + type=self.str(self.NormalizeWhitespace(typeRange)), + initializer=( + "" + if initializer is None + else self.str(self.NormalizeWhitespace(initializer)) + ), + initializerConstructorSyntax=constructorSyntax, + ) + + def ParseDeclaration(self, tokens: TokenRange): + nonWhitespace = lambda t: not t.IsWhitespace + initializer = None + beforeInitializer = tokens + constructorSyntax = False + equals = next( + ( + t + for t in AngleBracketParser.NotInsideAngleBrackets(tokens) + if t.Value == "=" and t.ParenDepth == tokens.First().ParenDepth + ), + None, + ) + if equals: + beforeInitializer = self.range(tokens.Begin, equals.Position) + initializer = self.range(equals.Position + 1, tokens.End) + else: + paren = next( + ( + t + for t in AngleBracketParser.NotInsideAngleBrackets(tokens) + if t.Value == "(" + ), + None, + ) + if paren: + constructorSyntax = True + beforeInitializer = self.range(tokens.Begin, paren.Position) + initializer = self.range(paren.Position + 1, tokens.End).TakeWhile( + lambda t, p=paren.ParenDepth: t.ParenDepth > p + ) + else: + brace = next( + ( + t + for t in AngleBracketParser.NotInsideAngleBrackets(tokens) + if t.Value == "{" + ), + None, + ) + if brace: + raise ActorCompilerError( + brace.SourceLine, + "Uniform initialization syntax is not currently supported for state variables (use '(' instead of '}' ?)", + ) + name = beforeInitializer.Last(nonWhitespace) + if beforeInitializer.Begin == name.Position: + raise ActorCompilerError( + beforeInitializer.First().SourceLine, "Declaration has no type." + ) + typeRange = self.range(beforeInitializer.Begin, name.Position) + return name, typeRange, initializer, constructorSyntax + + def NormalizeWhitespace(self, tokens: Iterable[Token]) -> Iterable[Token]: + inWhitespace = False + leading = True + for tok in tokens: + if not tok.IsWhitespace: + if inWhitespace and not leading: + yield Token(Value=" ") + inWhitespace = False + yield tok + leading = False + else: + inWhitespace = True + + def SplitParameterList( + self, toks: TokenRange, delimiter: str + ) -> Iterable[TokenRange]: + if toks.Begin == toks.End: + return [] + ranges: List[TokenRange] = [] + while True: + comma = next( + ( + t + for t in AngleBracketParser.NotInsideAngleBrackets(toks) + if t.Value == delimiter and t.ParenDepth == toks.First().ParenDepth + ), + None, + ) + if comma is None: + break + ranges.append(self.range(toks.Begin, comma.Position)) + toks = self.range(comma.Position + 1, toks.End) + ranges.append(toks) + return ranges + + def ParseLoopStatement(self, toks: TokenRange) -> LoopStatement: + return LoopStatement(body=self.ParseCompoundStatement(toks.Consume("loop"))) + + def ParseChooseStatement(self, toks: TokenRange) -> ChooseStatement: + return ChooseStatement(body=self.ParseCompoundStatement(toks.Consume("choose"))) + + def ParseWhenStatement(self, toks: TokenRange) -> WhenStatement: + expr = ( + toks.Consume("when") + .SkipWhile(lambda t: t.IsWhitespace) + .First() + .Assert("Expected (", lambda t: t.Value == "(") + .GetMatchingRangeIn(toks) + .SkipWhile(lambda t: t.IsWhitespace) + ) + return WhenStatement( + wait=self.ParseWaitStatement(expr), + body=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), + ) + + def ParseStateDeclaration(self, toks: TokenRange) -> StateDeclarationStatement: + toks = toks.Consume("state").RevSkipWhile(lambda t: t.Value == ";") + return StateDeclarationStatement(decl=self.ParseVarDeclaration(toks)) + + def ParseReturnStatement(self, toks: TokenRange) -> ReturnStatement: + toks = toks.Consume("return").RevSkipWhile(lambda t: t.Value == ";") + return ReturnStatement(expression=self.str(self.NormalizeWhitespace(toks))) + + def ParseThrowStatement(self, toks: TokenRange) -> ThrowStatement: + toks = toks.Consume("throw").RevSkipWhile(lambda t: t.Value == ";") + return ThrowStatement(expression=self.str(self.NormalizeWhitespace(toks))) + + def ParseWaitStatement(self, toks: TokenRange) -> WaitStatement: + ws = WaitStatement() + ws.FirstSourceLine = toks.First().SourceLine + if toks.First().Value == "state": + ws.resultIsState = True + toks = toks.Consume("state") + initializer = None + if toks.First().Value in ("wait", "waitNext"): + initializer = toks.RevSkipWhile(lambda t: t.Value == ";") + ws.result = VarDeclaration( + name="_", + type="Void", + initializer="", + initializerConstructorSyntax=False, + ) + else: + name, typeRange, initializer, constructorSyntax = self.ParseDeclaration( + toks.RevSkipWhile(lambda t: t.Value == ";") + ) + type_str = self.str(self.NormalizeWhitespace(typeRange)) + if type_str == "Void": + raise ActorCompilerError( + ws.FirstSourceLine, + "Assigning the result of a Void wait is not allowed. Just use a standalone wait statement.", + ) + ws.result = VarDeclaration( + name=name.Value, + type=self.str(self.NormalizeWhitespace(typeRange)), + initializer="", + initializerConstructorSyntax=False, + ) + if initializer is None: + raise ActorCompilerError( + ws.FirstSourceLine, + "Wait statement must be a declaration or standalone statement", + ) + waitParams = ( + initializer.SkipWhile(lambda t: t.IsWhitespace) + .Consume( + "Statement contains a wait, but is not a valid wait statement or a supported compound statement.1", + lambda t: True if t.Value in ("wait", "waitNext") else False, + ) + .SkipWhile(lambda t: t.IsWhitespace) + .First() + .Assert("Expected (", lambda t: t.Value == "(") + .GetMatchingRangeIn(initializer) + ) + if ( + not self.range(waitParams.End, initializer.End) + .Consume(")") + .All(lambda t: t.IsWhitespace) + ): + raise ActorCompilerError( + toks.First().SourceLine, + "Statement contains a wait, but is not a valid wait statement or a supported compound statement.2", + ) + ws.futureExpression = self.str(self.NormalizeWhitespace(waitParams)) + ws.isWaitNext = "waitNext" in [t.Value for t in initializer] + return ws + + def ParseWhileStatement(self, toks: TokenRange) -> WhileStatement: + expr = ( + toks.Consume("while") + .First(lambda t: not t.IsWhitespace) + .Assert("Expected (", lambda t: t.Value == "(") + .GetMatchingRangeIn(toks) + ) + return WhileStatement( + expression=self.str(self.NormalizeWhitespace(expr)), + body=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), + ) + + def ParseForStatement(self, toks: TokenRange) -> Statement: + head = ( + toks.Consume("for") + .First(lambda t: not t.IsWhitespace) + .Assert("Expected (", lambda t: t.Value == "(") + .GetMatchingRangeIn(toks) + ) + delim = [ + t + for t in head + if t.ParenDepth == head.First().ParenDepth + and t.BraceDepth == head.First().BraceDepth + and t.Value == ";" + ] + if len(delim) == 2: + init = self.range(head.Begin, delim[0].Position) + cond = self.range(delim[0].Position + 1, delim[1].Position) + next_expr = self.range(delim[1].Position + 1, head.End) + body = self.range(head.End + 1, toks.End) + return ForStatement( + initExpression=self.str(self.NormalizeWhitespace(init)), + condExpression=self.str(self.NormalizeWhitespace(cond)), + nextExpression=self.str(self.NormalizeWhitespace(next_expr)), + body=self.ParseCompoundStatement(body), + ) + delim = [ + t + for t in head + if t.ParenDepth == head.First().ParenDepth + and t.BraceDepth == head.First().BraceDepth + and t.Value == ":" + ] + if len(delim) != 1: + raise ActorCompilerError( + head.First().SourceLine, + "for statement must be 3-arg style or c++11 2-arg style", + ) + return RangeForStatement( + rangeExpression=self.str( + self.NormalizeWhitespace( + self.range(delim[0].Position + 1, head.End).SkipWhile( + lambda t: t.IsWhitespace + ) + ) + ), + rangeDecl=self.str( + self.NormalizeWhitespace( + self.range(head.Begin, delim[0].Position - 1).SkipWhile( + lambda t: t.IsWhitespace + ) + ) + ), + body=self.ParseCompoundStatement(self.range(head.End + 1, toks.End)), + ) + + def ParseIfStatement(self, toks: TokenRange) -> IfStatement: + toks = toks.Consume("if").SkipWhile(lambda t: t.IsWhitespace) + constexpr = toks.First().Value == "constexpr" + if constexpr: + toks = toks.Consume("constexpr").SkipWhile(lambda t: t.IsWhitespace) + expr = ( + toks.First(lambda t: not t.IsWhitespace) + .Assert("Expected (", lambda t: t.Value == "(") + .GetMatchingRangeIn(toks) + ) + return IfStatement( + expression=self.str(self.NormalizeWhitespace(expr)), + constexpr=constexpr, + ifBody=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), + ) + + def ParseElseStatement(self, toks: TokenRange, prev: Statement) -> None: + if_stmt = prev + while isinstance(if_stmt, IfStatement) and if_stmt.elseBody is not None: + if_stmt = if_stmt.elseBody + if not isinstance(if_stmt, IfStatement): + raise ActorCompilerError( + toks.First().SourceLine, "else without matching if" + ) + if_stmt.elseBody = self.ParseCompoundStatement(toks.Consume("else")) + + def ParseTryStatement(self, toks: TokenRange) -> TryStatement: + return TryStatement( + tryBody=self.ParseCompoundStatement(toks.Consume("try")), + catches=[], + ) + + def ParseCatchStatement(self, toks: TokenRange, prev: Statement) -> None: + if not isinstance(prev, TryStatement): + raise ActorCompilerError( + toks.First().SourceLine, "catch without matching try" + ) + expr = ( + toks.Consume("catch") + .First(lambda t: not t.IsWhitespace) + .Assert("Expected (", lambda t: t.Value == "(") + .GetMatchingRangeIn(toks) + ) + prev.catches.append( + TryStatement.Catch( + expression=self.str(self.NormalizeWhitespace(expr)), + body=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), + FirstSourceLine=expr.First().SourceLine, + ) + ) + + IllegalKeywords = {"goto", "do", "finally", "__if_exists", "__if_not_exists"} + + def ParseCompoundStatement(self, toks: TokenRange) -> Statement: + nonWhitespace = lambda t: not t.IsWhitespace + first = toks.First(nonWhitespace) + if first.Value == "{": + inBraces = first.GetMatchingRangeIn(toks) + if ( + not self.range(inBraces.End, toks.End) + .Consume("}") + .All(lambda t: t.IsWhitespace) + ): + raise ActorCompilerError( + inBraces.Last().SourceLine, + "Unexpected tokens after compound statement", + ) + return self.ParseCodeBlock(inBraces) + statements: List[Statement] = [] + self.ParseStatement(toks.Skip(1), statements) + return statements[0] + + def ParseCodeBlock(self, toks: TokenRange) -> CodeBlock: + statements: List[Statement] = [] + while True: + delim = next( + ( + t + for t in toks + if t.ParenDepth == toks.First().ParenDepth + and t.BraceDepth == toks.First().BraceDepth + and t.Value in (";", "}") + ), + None, + ) + if delim is None: + break + self.ParseStatement(self.range(toks.Begin, delim.Position + 1), statements) + toks = self.range(delim.Position + 1, toks.End) + if not toks.All(lambda t: t.IsWhitespace): + raise ActorCompilerError( + toks.First(lambda t: not t.IsWhitespace).SourceLine, + "Trailing unterminated statement in code block", + ) + return CodeBlock(statements=statements) + + def ParseStatement(self, toks: TokenRange, statements: List[Statement]) -> None: + nonWhitespace = lambda t: not t.IsWhitespace + toks = toks.SkipWhile(lambda t: t.IsWhitespace) + + def Add(stmt: Statement) -> None: + stmt.FirstSourceLine = toks.First().SourceLine + statements.append(stmt) + + first_val = toks.First().Value + if first_val == "loop": + Add(self.ParseLoopStatement(toks)) + elif first_val == "while": + Add(self.ParseWhileStatement(toks)) + elif first_val == "for": + Add(self.ParseForStatement(toks)) + elif first_val == "break": + Add(BreakStatement()) + elif first_val == "continue": + Add(ContinueStatement()) + elif first_val == "return": + Add(self.ParseReturnStatement(toks)) + elif first_val == "{": + Add(self.ParseCompoundStatement(toks)) + elif first_val == "if": + Add(self.ParseIfStatement(toks)) + elif first_val == "else": + self.ParseElseStatement(toks, statements[-1]) + elif first_val == "choose": + Add(self.ParseChooseStatement(toks)) + elif first_val == "when": + Add(self.ParseWhenStatement(toks)) + elif first_val == "try": + Add(self.ParseTryStatement(toks)) + elif first_val == "catch": + self.ParseCatchStatement(toks, statements[-1]) + elif first_val == "throw": + Add(self.ParseThrowStatement(toks)) + else: + if first_val in self.IllegalKeywords: + raise ActorCompilerError( + toks.First().SourceLine, + f"Statement '{first_val}' not supported in actors.", + ) + if any(t.Value in ("wait", "waitNext") for t in toks): + Add(self.ParseWaitStatement(toks)) + elif first_val == "state": + Add(self.ParseStateDeclaration(toks)) + elif first_val == "switch" and any(t.Value == "return" for t in toks): + raise ActorCompilerError( + toks.First().SourceLine, + "Unsupported compound statement containing return.", + ) + elif first_val.startswith("#"): + raise ActorCompilerError( + toks.First().SourceLine, + f'Found "{first_val}". Preprocessor directives are not supported within ACTORs', + ) + else: + cleaned = toks.RevSkipWhile(lambda t: t.Value == ";") + if any(nonWhitespace(t) for t in cleaned): + Add( + PlainOldCodeStatement( + code=self.str(self.NormalizeWhitespace(cleaned)) + ";" + ) + ) + + def ParseClassContext(self, toks: TokenRange) -> Tuple[bool, str]: + name = "" + if toks.Begin == toks.End: + return False, name + + nonWhitespace = lambda t: not t.IsWhitespace + while True: + first = toks.First(nonWhitespace) + if first.Value == "[": + contents = first.GetMatchingRangeIn(toks) + toks = self.range(contents.End + 1, toks.End) + elif first.Value == "alignas": + toks = self.range(first.Position + 1, toks.End) + first = toks.First(nonWhitespace) + first.Assert("Expected ( after alignas", lambda t: t.Value == "(") + contents = first.GetMatchingRangeIn(toks) + toks = self.range(contents.End + 1, toks.End) + else: + break + + first = toks.First(nonWhitespace) + if not self.identifierPattern.match(first.Value): + return False, name + + while True: + first.Assert( + "Expected identifier", lambda t: self.identifierPattern.match(t.Value) + ) + name += first.Value + toks = self.range(first.Position + 1, toks.End) + next_token = toks.First(nonWhitespace) + if next_token.Value == "::": + name += "::" + toks = toks.SkipWhile(lambda t: t.IsWhitespace).Skip(1) + else: + break + first = toks.First(nonWhitespace) + + toks = toks.SkipWhile( + lambda t: t.IsWhitespace or t.Value in ("final", "explicit") + ) + first = toks.First(nonWhitespace) + if first.Value in (":", "{"): + return True, name + return False, "" + + def str(self, tokens: Iterable[Token]) -> str: + return "".join(tok.Value for tok in tokens) diff --git a/flow/actorcompiler/actorcompiler.csproj b/flow/actorcompiler/actorcompiler.csproj deleted file mode 100644 index ae866447097..00000000000 --- a/flow/actorcompiler/actorcompiler.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - - Exe - net9.0 - false - true - - - diff --git a/flow/actorcompiler/main.py b/flow/actorcompiler/main.py new file mode 100644 index 00000000000..37a3f5a2d92 --- /dev/null +++ b/flow/actorcompiler/main.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import os +import stat +import sys +from pathlib import Path + +from . import ActorCompilerError +from .actor_parser import ActorParser, ErrorMessagePolicy + + +def overwrite_by_move(target: Path, temporary: Path) -> None: + if target.exists(): + target.chmod(stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + target.unlink() + os.replace(temporary, target) + target.chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + + +def parse_arguments(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="actorcompiler", + description="Python port of the Flow actor compiler", + add_help=False, + usage="actorcompiler [--disable-diagnostics] [--generate-probes]", + ) + parser.add_argument("input", nargs="?") + parser.add_argument("output", nargs="?") + parser.add_argument("--disable-diagnostics", action="store_true") + parser.add_argument("--generate-probes", action="store_true") + parser.add_argument("--help", action="help", help=argparse.SUPPRESS) + args = parser.parse_args(argv) + if not args.input or not args.output: + parser.print_usage(sys.stderr) + sys.exit(100) + return args + + +def main(argv: list[str] | None = None) -> int: + if argv is None: + argv = sys.argv[1:] + args = parse_arguments(argv) + input_path = Path(args.input) + output_path = Path(args.output) + output_tmp = output_path.with_suffix(output_path.suffix + ".tmp") + output_uid = output_path.with_suffix(output_path.suffix + ".uid") + + policy = ErrorMessagePolicy() + policy.DisableDiagnostics = args.disable_diagnostics + + try: + print("actorcompiler", " ".join(sys.argv[1:])) + text = input_path.read_text() + parser = ActorParser( + text, + str(input_path).replace("\\", "/"), + policy, + args.generate_probes, + ) + + with output_tmp.open("w", newline="\n") as out_file: + parser.Write(out_file, str(output_path).replace("\\", "/")) + overwrite_by_move(output_path, output_tmp) + + with output_tmp.open("w", newline="\n") as uid_file: + for (hi, lo), value in parser.uidObjects.items(): + uid_file.write(f"{hi}|{lo}|{value}\n") + overwrite_by_move(output_uid, output_tmp) + + return 0 + except ActorCompilerError as exc: + print( + f"{input_path}({exc.source_line}): error FAC1000: {exc}", + file=sys.stderr, + ) + if output_tmp.exists(): + output_tmp.unlink() + if output_path.exists(): + output_path.chmod(stat.S_IWUSR | stat.S_IRUSR) + output_path.unlink() + return 1 + except Exception as exc: # pylint: disable=broad-except + print( + f"{input_path}(1): error FAC2000: Internal {exc}", + file=sys.stderr, + ) + if output_tmp.exists(): + output_tmp.unlink() + if output_path.exists(): + output_path.chmod(stat.S_IWUSR | stat.S_IRUSR) + output_path.unlink() + return 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/flow/actorcompiler/parse_tree.py b/flow/actorcompiler/parse_tree.py new file mode 100644 index 00000000000..f1792eba07f --- /dev/null +++ b/flow/actorcompiler/parse_tree.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional, Sequence + + +@dataclass +class VarDeclaration: + type: str = "" + name: str = "" + initializer: Optional[str] = "" + initializerConstructorSyntax: bool = False + + +@dataclass +class Statement: + FirstSourceLine: int = 0 + + def containsWait(self) -> bool: + return False + + +@dataclass +class PlainOldCodeStatement(Statement): + code: str = "" + + def __str__(self) -> str: + return self.code + + +@dataclass +class StateDeclarationStatement(Statement): + decl: VarDeclaration = field(default_factory=VarDeclaration) + + def __str__(self) -> str: + if self.decl.initializerConstructorSyntax: + return f"State {self.decl.type} {self.decl.name}({self.decl.initializer});" + return f"State {self.decl.type} {self.decl.name} = {self.decl.initializer};" + + +@dataclass +class WhileStatement(Statement): + expression: str = "" + body: Statement = field(default_factory=Statement) + + def containsWait(self) -> bool: + return self.body.containsWait() + + +@dataclass +class ForStatement(Statement): + initExpression: str = "" + condExpression: str = "" + nextExpression: str = "" + body: Statement = field(default_factory=Statement) + + def containsWait(self) -> bool: + return self.body.containsWait() + + +@dataclass +class RangeForStatement(Statement): + rangeExpression: str = "" + rangeDecl: str = "" + body: Statement = field(default_factory=Statement) + + def containsWait(self) -> bool: + return self.body.containsWait() + + +@dataclass +class LoopStatement(Statement): + body: Statement = field(default_factory=Statement) + + def __str__(self) -> str: + return f"Loop {self.body}" + + def containsWait(self) -> bool: + return self.body.containsWait() + + +@dataclass +class BreakStatement(Statement): + pass + + +@dataclass +class ContinueStatement(Statement): + pass + + +@dataclass +class IfStatement(Statement): + expression: str = "" + constexpr: bool = False + ifBody: Statement = field(default_factory=Statement) + elseBody: Optional[Statement] = None + + def containsWait(self) -> bool: + return self.ifBody.containsWait() or ( + self.elseBody is not None and self.elseBody.containsWait() + ) + + +@dataclass +class ReturnStatement(Statement): + expression: str = "" + + def __str__(self) -> str: + return f"Return {self.expression}" + + +@dataclass +class WaitStatement(Statement): + result: VarDeclaration = field(default_factory=VarDeclaration) + futureExpression: str = "" + resultIsState: bool = False + isWaitNext: bool = False + + def __str__(self) -> str: + return f"Wait {self.result.type} {self.result.name} <- {self.futureExpression} ({'state' if self.resultIsState else 'local'})" + + def containsWait(self) -> bool: + return True + + +@dataclass +class ChooseStatement(Statement): + body: Statement = field(default_factory=Statement) + + def __str__(self) -> str: + return f"Choose {self.body}" + + def containsWait(self) -> bool: + return self.body.containsWait() + + +@dataclass +class WhenStatement(Statement): + wait: WaitStatement = field(default_factory=WaitStatement) + body: Optional[Statement] = None + + def __str__(self) -> str: + return f"When ({self.wait}) {self.body}" + + def containsWait(self) -> bool: + return True + + +@dataclass +class TryStatement(Statement): + @dataclass + class Catch: + expression: str = "" + body: Statement = field(default_factory=Statement) + FirstSourceLine: int = 0 + + tryBody: Statement = field(default_factory=Statement) + catches: List["TryStatement.Catch"] = field(default_factory=list) + + def containsWait(self) -> bool: + if self.tryBody.containsWait(): + return True + return any(c.body.containsWait() for c in self.catches) + + +@dataclass +class ThrowStatement(Statement): + expression: str = "" + + +@dataclass +class CodeBlock(Statement): + statements: Sequence[Statement] = field(default_factory=list) + + def __str__(self) -> str: + joined = "\n".join(str(stmt) for stmt in self.statements) + return f"CodeBlock\n{joined}\nEndCodeBlock" + + def containsWait(self) -> bool: + return any(stmt.containsWait() for stmt in self.statements) + + +@dataclass +class Declaration: + type: str = "" + name: str = "" + comment: str = "" + + +@dataclass +class Actor: + attributes: List[str] = field(default_factory=list) + returnType: Optional[str] = None + name: str = "" + enclosingClass: Optional[str] = None + parameters: Sequence[VarDeclaration] = field(default_factory=list) + templateFormals: Optional[Sequence[VarDeclaration]] = None + body: CodeBlock = field(default_factory=CodeBlock) + SourceLine: int = 0 + isStatic: bool = False + _isUncancellable: bool = False + testCaseParameters: Optional[str] = None + nameSpace: Optional[str] = None + isForwardDeclaration: bool = False + isTestCase: bool = False + + def IsCancellable(self) -> bool: + return self.returnType is not None and not self._isUncancellable + + def SetUncancellable(self) -> None: + self._isUncancellable = True + + +@dataclass +class Descr: + name: str = "" + superClassList: Optional[str] = None + body: List[Declaration] = field(default_factory=list) diff --git a/flow/actorcompiler/usertype.dat b/flow/actorcompiler/usertype.dat deleted file mode 100644 index 77a51990089..00000000000 --- a/flow/actorcompiler/usertype.dat +++ /dev/null @@ -1,8 +0,0 @@ -ACTOR -loop -state -choose -when -wait -waitNext -DESCR \ No newline at end of file From 49c4fbf7295552dbdf466696a83cbab2c2f15a3e Mon Sep 17 00:00:00 2001 From: Hendrik Hofstadt Date: Wed, 19 Nov 2025 13:25:16 +0000 Subject: [PATCH 2/2] Address review and restore C# compiler --- cmake/CompileActorCompiler.cmake | 72 +- cmake/FlowCommands.cmake | 23 +- flow/actorcompiler/ActorCompiler.cs | 1476 +++++++++++++++++ flow/actorcompiler/ActorCompiler.targets | 54 + flow/actorcompiler/ActorCompiler.xml | 62 + flow/actorcompiler/ActorParser.cs | 1178 +++++++++++++ flow/actorcompiler/ParseTree.cs | 252 +++ flow/actorcompiler/Program.cs | 104 ++ flow/actorcompiler/Properties/AssemblyInfo.cs | 56 + flow/actorcompiler/__main__.py | 10 - flow/actorcompiler/actor_parser.py | 1134 ------------- flow/actorcompiler/actorcompiler.csproj | 10 + flow/actorcompiler/parse_tree.py | 219 --- flow/actorcompiler/usertype.dat | 8 + flow/actorcompiler_py/Actor checklist.txt | 84 + .../main.py => actorcompiler_py/__main__.py} | 24 +- .../actor_compiler.py | 894 +++++----- flow/actorcompiler_py/actor_parser.py | 1126 +++++++++++++ flow/actorcompiler_py/compare_actor_output.py | 60 + .../errors.py} | 20 +- flow/actorcompiler_py/parse_tree.py | 235 +++ 21 files changed, 5250 insertions(+), 1851 deletions(-) create mode 100644 flow/actorcompiler/ActorCompiler.cs create mode 100644 flow/actorcompiler/ActorCompiler.targets create mode 100644 flow/actorcompiler/ActorCompiler.xml create mode 100644 flow/actorcompiler/ActorParser.cs create mode 100644 flow/actorcompiler/ParseTree.cs create mode 100644 flow/actorcompiler/Program.cs create mode 100644 flow/actorcompiler/Properties/AssemblyInfo.cs delete mode 100644 flow/actorcompiler/__main__.py delete mode 100644 flow/actorcompiler/actor_parser.py create mode 100644 flow/actorcompiler/actorcompiler.csproj delete mode 100644 flow/actorcompiler/parse_tree.py create mode 100644 flow/actorcompiler/usertype.dat create mode 100644 flow/actorcompiler_py/Actor checklist.txt rename flow/{actorcompiler/main.py => actorcompiler_py/__main__.py} (84%) rename flow/{actorcompiler => actorcompiler_py}/actor_compiler.py (63%) create mode 100644 flow/actorcompiler_py/actor_parser.py create mode 100644 flow/actorcompiler_py/compare_actor_output.py rename flow/{actorcompiler/__init__.py => actorcompiler_py/errors.py} (51%) create mode 100644 flow/actorcompiler_py/parse_tree.py diff --git a/cmake/CompileActorCompiler.cmake b/cmake/CompileActorCompiler.cmake index 5443a82e33c..1e3c12d19f6 100644 --- a/cmake/CompileActorCompiler.cmake +++ b/cmake/CompileActorCompiler.cmake @@ -1,14 +1,72 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter) +find_program(MCS_EXECUTABLE mcs) +find_program(MONO_EXECUTABLE mono) + +set(ACTORCOMPILER_PY_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/__main__.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/errors.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/actor_parser.py + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler_py/actor_compiler.py) + +set(ACTORCOMPILER_CSPROJ + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actorcompiler.csproj) + set(ACTORCOMPILER_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/__init__.py - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/__main__.py - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/main.py - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actor_parser.py - ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/actor_compiler.py) + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorCompiler.cs + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ActorParser.cs + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/ParseTree.cs + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Program.cs + ${CMAKE_CURRENT_SOURCE_DIR}/flow/actorcompiler/Properties/AssemblyInfo.cs) + +set(ACTOR_COMPILER_REFERENCES + "-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml" +) + +add_custom_target(actorcompiler_py DEPENDS ${ACTORCOMPILER_PY_SRCS}) -add_custom_target(actorcompiler DEPENDS ${ACTORCOMPILER_SRCS}) +if(WIN32) + add_executable(actorcompiler_csharp ${ACTORCOMPILER_SRCS}) + target_compile_options(actorcompiler_csharp PRIVATE "/langversion:6") + set_property( + TARGET actorcompiler_csharp + PROPERTY VS_DOTNET_REFERENCES + "System" + "System.Core" + "System.Xml.Linq" + "System.Data.DataSetExtensions" + "Microsoft.CSharp" + "System.Data" + "System.Xml") + set(ACTORCOMPILER_CSHARP_COMMAND $ + CACHE INTERNAL "Command to run the C# actor compiler") + add_custom_target(actorcompiler) + add_dependencies(actorcompiler actorcompiler_csharp actorcompiler_py) +elseif(CSHARP_USE_MONO) + add_custom_command( + OUTPUT actorcompiler.exe + COMMAND ${CSHARP_COMPILER_EXECUTABLE} ARGS ${ACTOR_COMPILER_REFERENCES} + ${ACTORCOMPILER_SRCS} "-target:exe" "-out:actorcompiler.exe" + DEPENDS ${ACTORCOMPILER_SRCS} + COMMENT "Compile actor compiler" + VERBATIM) + add_custom_target(actorcompiler_csharp + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe) + set(actor_exe "${CMAKE_CURRENT_BINARY_DIR}/actorcompiler.exe") + set(ACTORCOMPILER_CSHARP_COMMAND ${MONO_EXECUTABLE} ${actor_exe} + CACHE INTERNAL "Command to run the C# actor compiler") + add_custom_target(actorcompiler) + add_dependencies(actorcompiler actorcompiler_csharp actorcompiler_py) +else() + dotnet_build(${ACTORCOMPILER_CSPROJ} SOURCE ${ACTORCOMPILER_SRCS}) + set(actor_exe "${actorcompiler_EXECUTABLE_PATH}") + message(STATUS "Actor compiler path: ${actor_exe}") + # dotnet_build already creates a target named 'actorcompiler', so we just add Python dependency + add_dependencies(actorcompiler actorcompiler_py) + set(ACTORCOMPILER_CSHARP_COMMAND ${actor_exe} + CACHE INTERNAL "Command to run the C# actor compiler") +endif() set(ACTORCOMPILER_COMMAND - ${Python3_EXECUTABLE} -m flow.actorcompiler + ${Python3_EXECUTABLE} -m flow.actorcompiler_py CACHE INTERNAL "Command to run the actor compiler") diff --git a/cmake/FlowCommands.cmake b/cmake/FlowCommands.cmake index e0f77908fbe..cee4727871c 100644 --- a/cmake/FlowCommands.cmake +++ b/cmake/FlowCommands.cmake @@ -263,11 +263,24 @@ function(add_flow_target) endforeach() list(APPEND generated_files ${out_file}) - add_custom_command(OUTPUT "${out_file}" - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CMAKE_SOURCE_DIR}" - ${ACTORCOMPILER_COMMAND} "${in_file}" "${out_file}" ${actor_compiler_flags} - DEPENDS "${in_file}" actorcompiler - COMMENT "Compile actor: ${src}") + if(ACTORCOMPILER_CSHARP_COMMAND) + set(py_out_file "${out_file}.py_gen") + set(cs_out_file "${out_file}.cs_gen") + add_custom_command(OUTPUT "${out_file}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CMAKE_SOURCE_DIR}" + ${ACTORCOMPILER_COMMAND} "${in_file}" "${py_out_file}" ${actor_compiler_flags} + COMMAND ${ACTORCOMPILER_CSHARP_COMMAND} "${in_file}" "${cs_out_file}" ${actor_compiler_flags} + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/flow/actorcompiler_py/compare_actor_output.py "${cs_out_file}" "${py_out_file}" + COMMAND ${CMAKE_COMMAND} -E copy "${py_out_file}" "${out_file}" + DEPENDS "${in_file}" actorcompiler + COMMENT "Compile and compare actor: ${src}") + else() + add_custom_command(OUTPUT "${out_file}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CMAKE_SOURCE_DIR}" + ${ACTORCOMPILER_COMMAND} "${in_file}" "${out_file}" ${actor_compiler_flags} + DEPENDS "${in_file}" actorcompiler + COMMENT "Compile actor: ${src}") + endif() endif() endforeach() if(PASS_COMPILATION_UNIT) diff --git a/flow/actorcompiler/ActorCompiler.cs b/flow/actorcompiler/ActorCompiler.cs new file mode 100644 index 00000000000..ab906d47d5d --- /dev/null +++ b/flow/actorcompiler/ActorCompiler.cs @@ -0,0 +1,1476 @@ +/* + * ActorCompiler.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Security.Cryptography; + +namespace actorcompiler +{ + class TypeSwitch + { + object value; + R result; + bool ok = false; + public TypeSwitch(object t) { this.value = t; } + public TypeSwitch Case(Func f) + where T : class + { + if (!ok) + { + var t = value as T; + if (t != null) + { + result = f(t); + ok = true; + } + } + return this; + } + public R Return() + { + if (!ok) throw new Exception("Typeswitch didn't match."); + return result; + } + } + + class Context + { + public Function target; + public Function next; + public Function breakF; + public Function continueF; + public Function catchFErr; // Catch function taking Error + public int tryLoopDepth = 0; // The number of (loopDepth-increasing) loops entered inside the innermost try (thus, that will be exited by a throw) + + public void unreachable() { target = null; } + + // Usually we just change the target of a context + public Context WithTarget(Function newTarget) { return new Context { target = newTarget, breakF = breakF, continueF = continueF, next = null, catchFErr = catchFErr, tryLoopDepth = tryLoopDepth }; } + + // When entering a loop, we have to provide new break and continue functions + public Context LoopContext(Function newTarget, Function breakF, Function continueF, int deltaLoopDepth) { return new Context { target = newTarget, breakF = breakF, continueF = continueF, next = null, catchFErr = catchFErr, tryLoopDepth = tryLoopDepth + deltaLoopDepth }; } + + public Context WithCatch(Function newCatchFErr) { return new Context { target = target, breakF = breakF, continueF = continueF, next = null, catchFErr = newCatchFErr, tryLoopDepth = 0 }; } + + public Context Clone() { return new Context { target = target, next = next, breakF = breakF, continueF = continueF, catchFErr = catchFErr, tryLoopDepth = tryLoopDepth }; } + }; + + class Function + { + public string name; + public string returnType; + public string[] formalParameters; + public bool endIsUnreachable = false; + public string exceptionParameterIs = null; + public bool publicName = false; + public string specifiers; + string indentation; + StreamWriter body; + public bool wasCalled { get; protected set; } + public Function overload = null; + + public Function() + { + body = new StreamWriter(new MemoryStream()); + } + + public void setOverload(Function overload) { + this.overload = overload; + } + + public Function popOverload() { + Function result = this.overload; + this.overload = null; + return result; + } + + public void addOverload(params string[] formalParameters) { + setOverload( + new Function { + name = name, + returnType = returnType, + endIsUnreachable = endIsUnreachable, + formalParameters = formalParameters, + indentation = indentation + } + ); + } + + public void Indent(int change) + { + for(int i=0; i0, run the actor beginning at point P until + * (1) it waits, in which case set an appropriate callback and return 0, or + * (2) it returns, in which case destroy the actor and return 0, or + * (3) it reaches the bottom of the Nth innermost loop containing P, in which case + * return max(0, the given loopDepth - N) (N=0 for the innermost loop, N=1 for the next innermost, etc) + * + * Examples: + * Source: + * loop + * [P] + * loop + * [P'] + * loop + * [P''] + * break + * [Q'] + * break + * [Q] + * + * fP(1) should execute everything from [P] to [Q] and then return 1 (since [Q] is at the bottom of the 0th innermost loop containing [P]) + * fP'(2) should execute everything from [P'] to [Q] and then return 1 (since [Q] is at the bottom of the 1st innermost loop containing [P']) + * fP''(3) should execute everything from [P''] to [Q] and then return 1 (since [Q] is at the bottom of the 2nd innermost loop containing [P'']) + * fQ'(2) should execute everything from [Q'] to [Q] and then return 1 (since [Q] is at the bottom of the 1st innermost loop containing [Q']) + * fQ(1) should return 1 (since [Q] is at the bottom of the 0th innermost loop containing [Q]) + */ + }; + + class LiteralBreak : Function + { + public LiteralBreak() { name = "break!"; } + public override string call(params string[] parameters) + { + wasCalled = true; + if (parameters.Length != 0) throw new Exception("LiteralBreak called with parameters!"); + return "break"; + } + }; + + class LiteralContinue : Function + { + public LiteralContinue() { name = "continue!"; } + public override string call(params string[] parameters) + { + wasCalled = true; + if (parameters.Length != 0) throw new Exception("LiteralContinue called with parameters!"); + return "continue"; + } + }; + + class StateVar : VarDeclaration + { + public int SourceLine; + }; + + class CallbackVar : StateVar + { + public int CallbackGroup; + } + + class DescrCompiler + { + Descr descr; + string memberIndentStr; + + public DescrCompiler(Descr descr, int braceDepth) + { + this.descr = descr; + this.memberIndentStr = new string('\t', braceDepth); + } + public void Write(TextWriter writer, out int lines) + { + lines = 0; + + writer.WriteLine(memberIndentStr + "template<> struct Descriptor {{", descr.name); + writer.WriteLine(memberIndentStr + "\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", descr.name); + writer.WriteLine(memberIndentStr + "\ttypedef {0} type;", descr.name); + lines += 3; + + foreach (var dec in descr.body) + { + writer.WriteLine(memberIndentStr + "\tstruct {0}Descriptor {{", dec.name); + writer.WriteLine(memberIndentStr + "\t\tstatic StringRef name() {{ return \"{0}\"_sr; }}", dec.name); + writer.WriteLine(memberIndentStr + "\t\tstatic StringRef typeName() {{ return \"{0}\"_sr; }}", dec.type); + writer.WriteLine(memberIndentStr + "\t\tstatic StringRef comment() {{ return \"{0}\"_sr; }}", dec.comment); + writer.WriteLine(memberIndentStr + "\t\ttypedef {0} type;", dec.type); + writer.WriteLine(memberIndentStr + "\t\tstatic inline type get({0}& from);", descr.name); + writer.WriteLine(memberIndentStr + "\t};"); + lines += 7; + } + + writer.Write(memberIndentStr + "\ttypedef std::tuple<"); + bool FirstDesc = true; + foreach (var dec in descr.body) + { + if (!FirstDesc) + writer.Write(","); + writer.Write("{0}Descriptor", dec.name); + FirstDesc = false; + } + writer.Write("> fields;\n"); + writer.WriteLine(memberIndentStr + "\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type field_indexes;"); + writer.WriteLine(memberIndentStr + "};"); + if(descr.superClassList != null) + writer.WriteLine(memberIndentStr + "struct {0} : {1} {{", descr.name, descr.superClassList); + else + writer.WriteLine(memberIndentStr + "struct {0} {{", descr.name); + lines += 4; + + foreach (var dec in descr.body) + { + writer.WriteLine(memberIndentStr + "\t{0} {1}; //{2}", dec.type, dec.name, dec.comment); + lines++; + } + + writer.WriteLine(memberIndentStr + "};"); + lines++; + + foreach (var dec in descr.body) + { + writer.WriteLine(memberIndentStr + "{0} Descriptor<{1}>::{2}Descriptor::get({1}& from) {{ return from.{2}; }}", dec.type, descr.name, dec.name); + lines++; + } + + } + } + + class ActorCompiler + { + Actor actor; + string className, fullClassName, stateClassName; + string sourceFile; + List state; + List callbacks = new List(); + bool isTopLevel; + const string loopDepth0 = "int loopDepth=0"; + const string loopDepth = "int loopDepth"; + const int codeIndent = +2; + const string memberIndentStr = "\t"; + static HashSet usedClassNames = new HashSet(); + bool LineNumbersEnabled; + int chooseGroups = 0, whenCount = 0; + string This; + bool generateProbes; + public Dictionary<(ulong, ulong), string> uidObjects { get; private set; } + + public ActorCompiler(Actor actor, string sourceFile, bool isTopLevel, bool lineNumbersEnabled, bool generateProbes) + { + this.actor = actor; + this.sourceFile = sourceFile; + this.isTopLevel = isTopLevel; + this.LineNumbersEnabled = lineNumbersEnabled; + this.generateProbes = generateProbes; + this.uidObjects = new Dictionary<(ulong, ulong), string>(); + + FindState(); + } + + private ulong ByteToLong(byte[] bytes) { + // NOTE: Always assume big endian. + ulong result = 0; + foreach(var b in bytes) { + result += b; + result <<= 8; + } + return result; + } + + // Generates the identifier for the ACTOR + private Tuple GetUidFromString(string str) { + byte[] sha256Hash = SHA256.Create().ComputeHash(Encoding.UTF8.GetBytes(str)); + byte[] first = sha256Hash.Take(8).ToArray(); + byte[] second = sha256Hash.Skip(8).Take(8).ToArray(); + return new Tuple(ByteToLong(first), ByteToLong(second)); + } + + // Writes the function that returns the Actor object + private void WriteActorFunction(TextWriter writer, string fullReturnType) { + WriteTemplate(writer); + LineNumber(writer, actor.SourceLine); + foreach (string attribute in actor.attributes) { + writer.Write(attribute + " "); + } + if (actor.isStatic) writer.Write("static "); + writer.WriteLine("{0} {3}{1}( {2} ) {{", fullReturnType, actor.name, string.Join(", ", ParameterList()), actor.nameSpace==null ? "" : actor.nameSpace + "::"); + LineNumber(writer, actor.SourceLine); + + string newActor = string.Format("new {0}({1})", + fullClassName, + string.Join(", ", actor.parameters.Select(p => p.name).ToArray())); + + if (actor.returnType != null) + writer.WriteLine("\treturn Future<{1}>({0});", newActor, actor.returnType); + else + writer.WriteLine("\t{0};", newActor); + writer.WriteLine("}"); + } + + // Writes the class of the Actor object + private void WriteActorClass(TextWriter writer, string fullStateClassName, Function body) { + // The final actor class mixes in the State class, the Actor base class and all callback classes + writer.WriteLine("// This generated class is to be used only via {0}()", actor.name); + WriteTemplate(writer); + LineNumber(writer, actor.SourceLine); + + string callback_base_classes = string.Join(", ", callbacks.Select(c=>string.Format("public {0}", c.type))); + if (callback_base_classes != "") callback_base_classes += ", "; + writer.WriteLine("class {0} final : public Actor<{2}>, {3}public FastAllocated<{1}>, public {4} {{", + className, + fullClassName, + actor.returnType == null ? "void" : actor.returnType, + callback_base_classes, + fullStateClassName + ); + writer.WriteLine("public:"); + writer.WriteLine("\tusing FastAllocated<{0}>::operator new;", fullClassName); + writer.WriteLine("\tusing FastAllocated<{0}>::operator delete;", fullClassName); + + var actorIdentifierKey = this.sourceFile + ":" + this.actor.name; + var actorIdentifier = GetUidFromString(actorIdentifierKey); + uidObjects.Add((actorIdentifier.Item1, actorIdentifier.Item2), actorIdentifierKey); + // NOTE UL is required as a u64 postfix for large integers, otherwise Clang would complain + writer.WriteLine("\tstatic constexpr ActorIdentifier __actorIdentifier = UID({0}UL, {1}UL);", actorIdentifier.Item1, actorIdentifier.Item2); + writer.WriteLine("\tActiveActorHelper activeActorHelper;"); + + writer.WriteLine("#pragma clang diagnostic push"); + writer.WriteLine("#pragma clang diagnostic ignored \"-Wdelete-non-virtual-dtor\""); + if (actor.returnType != null) + writer.WriteLine(@" void destroy() override {{ + activeActorHelper.~ActiveActorHelper(); + static_cast*>(this)->~Actor(); + operator delete(this); + }}", actor.returnType); + else + writer.WriteLine(@" void destroy() {{ + activeActorHelper.~ActiveActorHelper(); + static_cast*>(this)->~Actor(); + operator delete(this); + }}"); + writer.WriteLine("#pragma clang diagnostic pop"); + + foreach (var cb in callbacks) + writer.WriteLine("friend struct {0};", cb.type); + + LineNumber(writer, actor.SourceLine); + WriteConstructor(body, writer, fullStateClassName); + WriteCancelFunc(writer); + writer.WriteLine("};"); + + } + + public void Write(TextWriter writer) + { + string fullReturnType = + actor.returnType != null ? string.Format("Future<{0}>", actor.returnType) + : "void"; + for (int i = 0; ; i++) + { + className = string.Format("{3}{0}{1}Actor{2}", + actor.name.Substring(0, 1).ToUpper(), + actor.name.Substring(1), + i != 0 ? i.ToString() : "", + actor.enclosingClass != null && actor.isForwardDeclaration ? actor.enclosingClass.Replace("::", "_") + "_" + : actor.nameSpace != null ? actor.nameSpace.Replace("::", "_") + "_" + : ""); + if (actor.isForwardDeclaration || usedClassNames.Add(className)) + break; + } + + // e.g. SimpleTimerActor + fullClassName = className + GetTemplateActuals(); + var actorClassFormal = new VarDeclaration { name = className, type = "class" }; + This = string.Format("static_cast<{0}*>(this)", actorClassFormal.name); + // e.g. SimpleTimerActorState + stateClassName = className + "State"; + // e.g. SimpleTimerActorState + var fullStateClassName = stateClassName + GetTemplateActuals(new VarDeclaration { type = "class", name = fullClassName }); + + if (actor.isForwardDeclaration) { + foreach (string attribute in actor.attributes) { + writer.Write(attribute + " "); + } + if (actor.isStatic) writer.Write("static "); + writer.WriteLine("{0} {3}{1}( {2} );", fullReturnType, actor.name, string.Join(", ", ParameterList()), actor.nameSpace==null ? "" : actor.nameSpace + "::"); + if (actor.enclosingClass != null) { + writer.WriteLine("template friend class {0};", stateClassName); + } + return; + } + + var body = getFunction("", "body", loopDepth0); + var bodyContext = new Context { + target = body, + catchFErr = getFunction(body.name, "Catch", "Error error", loopDepth0), + }; + + var endContext = TryCatchCompile(actor.body, bodyContext); + + if (endContext.target != null) + { + if (actor.returnType == null) + CompileStatement(new ReturnStatement { FirstSourceLine = actor.SourceLine, expression = "" }, endContext ); + else + throw new Error(actor.SourceLine, "Actor {0} fails to return a value", actor.name); + } + + if (actor.returnType != null) + { + bodyContext.catchFErr.WriteLine("this->~{0}();", stateClassName); + bodyContext.catchFErr.WriteLine("{0}->sendErrorAndDelPromiseRef(error);", This); + } + else + { + bodyContext.catchFErr.WriteLine("delete {0};", This); + } + bodyContext.catchFErr.WriteLine("loopDepth = 0;"); + + if (isTopLevel && actor.nameSpace == null) writer.WriteLine("namespace {"); + + // The "State" class contains all state and user code, to make sure that state names are accessible to user code but + // inherited members of Actor, Callback etc are not. + writer.WriteLine("// This generated class is to be used only via {0}()", actor.name); + WriteTemplate(writer, actorClassFormal); + LineNumber(writer, actor.SourceLine); + writer.WriteLine("class {0} {{", stateClassName); + writer.WriteLine("public:"); + LineNumber(writer, actor.SourceLine); + WriteStateConstructor(writer); + WriteStateDestructor(writer); + WriteFunctions(writer); + foreach (var st in state) + { + LineNumber(writer, st.SourceLine); + writer.WriteLine("\t{0} {1};", st.type, st.name); + } + writer.WriteLine("};"); + + WriteActorClass(writer, fullStateClassName, body); + + if (isTopLevel && actor.nameSpace == null) writer.WriteLine("} // namespace"); // namespace + + WriteActorFunction(writer, fullReturnType); + + if (actor.testCaseParameters != null) + { + writer.WriteLine("ACTOR_TEST_CASE({0}, {1})", actor.name, actor.testCaseParameters); + } + + Console.WriteLine("\tCompiled ACTOR {0} (line {1})", actor.name, actor.SourceLine); + } + + const string thisAddress = "reinterpret_cast(this)"; + + void ProbeEnter(Function fun, string name, int index = -1) { + if (generateProbes) { + fun.WriteLine("fdb_probe_actor_enter(\"{0}\", {1}, {2});", name, thisAddress, index); + } + var blockIdentifier = GetUidFromString(fun.name); + fun.WriteLine("#ifdef WITH_ACAC"); + fun.WriteLine("static constexpr ActorBlockIdentifier __identifier = UID({0}UL, {1}UL);", blockIdentifier.Item1, blockIdentifier.Item2); + fun.WriteLine("ActorExecutionContextHelper __helper(static_cast<{0}*>(this)->activeActorHelper.actorID, __identifier);", className); + fun.WriteLine("#endif // WITH_ACAC"); + } + + void ProbeExit(Function fun, string name, int index = -1) { + if (generateProbes) { + fun.WriteLine("fdb_probe_actor_exit(\"{0}\", {1}, {2});", name, thisAddress, index); + } + } + + void ProbeCreate(Function fun, string name) { + if (generateProbes) { + fun.WriteLine("fdb_probe_actor_create(\"{0}\", {1});", name, thisAddress); + } + } + + void ProbeDestroy(Function fun, string name) { + if (generateProbes) { + fun.WriteLine("fdb_probe_actor_destroy(\"{0}\", {1});", name, thisAddress); + } + } + + void LineNumber(TextWriter writer, int SourceLine) + { + if(SourceLine == 0) + { + throw new Exception("Internal error: Invalid source line (0)"); + } + if (LineNumbersEnabled) + writer.WriteLine("\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} \"{1}\"", SourceLine, sourceFile); + } + void LineNumber(Function writer, int SourceLine) + { + if(SourceLine == 0) + { + throw new Exception("Internal error: Invalid source line (0)"); + } + if (LineNumbersEnabled) + writer.WriteLineUnindented( string.Format("\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} \"{1}\"", SourceLine, sourceFile) ); + } + + void TryCatch(Context cx, Function catchFErr, int catchLoopDepth, Action action, bool useLoopDepth = true) + { + if (catchFErr!=null) + { + cx.target.WriteLine("try {"); + cx.target.Indent(+1); + } + action(); + if (catchFErr!=null) + { + cx.target.Indent(-1); + cx.target.WriteLine("}"); + + cx.target.WriteLine("catch (Error& error) {"); + if (useLoopDepth) + cx.target.WriteLine("\tloopDepth = {0};", catchFErr.call("error", AdjustLoopDepth(catchLoopDepth))); + else + cx.target.WriteLine("\t{0};", catchFErr.call("error", "0")); + cx.target.WriteLine("} catch (...) {"); + if (useLoopDepth) + cx.target.WriteLine("\tloopDepth = {0};", catchFErr.call("unknown_error()", AdjustLoopDepth(catchLoopDepth))); + else + cx.target.WriteLine("\t{0};", catchFErr.call("unknown_error()", "0")); + cx.target.WriteLine("}"); + } + } + Context TryCatchCompile(CodeBlock block, Context cx) + { + TryCatch(cx, cx.catchFErr, cx.tryLoopDepth, () => { + cx = Compile(block, cx, true); + if (cx.target != null) + { + var next = getFunction(cx.target.name, "cont", loopDepth); + cx.target.WriteLine("loopDepth = {0};", next.call("loopDepth")); + cx.target = next; + cx.next = null; + } + }); + return cx; + } + + void WriteTemplate(TextWriter writer, params VarDeclaration[] extraParameters) + { + var formals = (actor.templateFormals!=null ? actor.templateFormals.AsEnumerable() : Enumerable.Empty()) + .Concat(extraParameters) + .ToArray(); + + if (formals.Length==0) return; + LineNumber(writer, actor.SourceLine); + writer.WriteLine("template <{0}>", + string.Join(", ", formals.Select( + p => string.Format("{0} {1}", p.type, p.name) + ).ToArray())); + } + string GetTemplateActuals(params VarDeclaration[] extraParameters) + { + var formals = (actor.templateFormals != null ? actor.templateFormals.AsEnumerable() : Enumerable.Empty()) + .Concat(extraParameters) + .ToArray(); + + if (formals.Length == 0) return ""; + else return "<" + + string.Join(", ", formals.Select( + p => p.name + ).ToArray()) + + ">"; + } + + bool WillContinue(Statement stmt) + { + return Flatten(stmt).Any( + st => (st is ChooseStatement || st is WaitStatement || st is TryStatement)); + } + + CodeBlock AsCodeBlock(Statement statement) + { + // SOMEDAY: Is this necessary? Maybe we should just be compiling statements? + var cb = statement as CodeBlock; + if (cb != null) return cb; + return new CodeBlock { statements = new Statement[] { statement } }; + } + + void CompileStatement(PlainOldCodeStatement stmt, Context cx) + { + LineNumber(cx.target,stmt.FirstSourceLine); + cx.target.WriteLine(stmt.code); + } + void CompileStatement(StateDeclarationStatement stmt, Context cx) + { + // if this state declaration is at the very top of the actor body + if (actor.body.statements.Select(x => x as StateDeclarationStatement).TakeWhile(x => x != null).Any(x => x == stmt)) + { + // Initialize the state in the constructor, not here + state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, name = stmt.decl.name, type = stmt.decl.type, + initializer = stmt.decl.initializer, initializerConstructorSyntax = stmt.decl.initializerConstructorSyntax } ); + } + else + { + // State variables declared elsewhere must have a default constructor + state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, name = stmt.decl.name, type = stmt.decl.type, initializer = null }); + if (stmt.decl.initializer != null) + { + LineNumber(cx.target, stmt.FirstSourceLine); + if (stmt.decl.initializerConstructorSyntax || stmt.decl.initializer=="") + cx.target.WriteLine("{0} = {1}({2});", stmt.decl.name, stmt.decl.type, stmt.decl.initializer); + else + cx.target.WriteLine("{0} = {1};", stmt.decl.name, stmt.decl.initializer); + } + } + } + + void CompileStatement(ForStatement stmt, Context cx) + { + // for( initExpression; condExpression; nextExpression ) body; + + bool noCondition = stmt.condExpression == "" || stmt.condExpression == "true" || stmt.condExpression == "1"; + + if (!WillContinue(stmt.body)) + { + // We can write this loop without anything fancy, because there are no wait statements in it + if (EmitNativeLoop(stmt.FirstSourceLine, "for(" + stmt.initExpression + ";" + stmt.condExpression + ";" + stmt.nextExpression+")", stmt.body, cx) + && noCondition ) + cx.unreachable(); + } + else + { + // First compile the initExpression + CompileStatement( new PlainOldCodeStatement {code = stmt.initExpression + ";", FirstSourceLine = stmt.FirstSourceLine}, cx ); + + // fullBody = { if (!(condExpression)) break; body; } + Statement fullBody = noCondition ? stmt.body : + new CodeBlock + { + statements = new Statement[] { + new IfStatement + { + expression = "!(" + stmt.condExpression + ")", + ifBody = new BreakStatement { FirstSourceLine = stmt.FirstSourceLine }, + FirstSourceLine = stmt.FirstSourceLine + } + }.Concat(AsCodeBlock(stmt.body).statements).ToArray(), + FirstSourceLine = stmt.FirstSourceLine + }; + + Function loopF = getFunction(cx.target.name, "loopHead", loopDepth); + Function loopBody = getFunction(cx.target.name, "loopBody", loopDepth); + Function breakF = getFunction(cx.target.name, "break", loopDepth); + Function continueF = stmt.nextExpression == "" ? loopF : getFunction(cx.target.name, "continue", loopDepth); + + // TODO: Could we use EmitNativeLoop() here? + + loopF.WriteLine("int oldLoopDepth = ++loopDepth;"); + loopF.WriteLine("while (loopDepth == oldLoopDepth) loopDepth = {0};", loopBody.call("loopDepth")); + + Function endLoop = Compile(AsCodeBlock(fullBody), cx.LoopContext( loopBody, breakF, continueF, +1 ), true).target; + if (endLoop != null && endLoop != loopBody) { + if (stmt.nextExpression != "") + CompileStatement( new PlainOldCodeStatement {code = stmt.nextExpression + ";", FirstSourceLine = stmt.FirstSourceLine}, cx.WithTarget(endLoop) ); + endLoop.WriteLine("if (loopDepth == 0) return {0};", loopF.call("0")); + } + + cx.target.WriteLine("loopDepth = {0};", loopF.call("loopDepth")); + + if (continueF != loopF && continueF.wasCalled) { + CompileStatement( new PlainOldCodeStatement {code = stmt.nextExpression + ";", FirstSourceLine = stmt.FirstSourceLine}, cx.WithTarget(continueF) ); + continueF.WriteLine("if (loopDepth == 0) return {0};", loopF.call("0")); + } + + if (breakF.wasCalled) + TryCatch(cx.WithTarget(breakF), cx.catchFErr, cx.tryLoopDepth, + () => { breakF.WriteLine("return {0};", cx.next.call("loopDepth")); }); + else + cx.unreachable(); + } + } + + Dictionary iterators = new Dictionary(); + + string getIteratorName(Context cx) + { + string name = "RangeFor" + cx.target.name + "Iterator"; + if (!iterators.ContainsKey(name)) + iterators[name] = 0; + return string.Format("{0}{1}", name, iterators[name]++); + } + + void CompileStatement(RangeForStatement stmt, Context cx) + { + // If stmt does not contain a wait statement, rewrite the original c++11 range-based for loop + // If there is a wait, we need to rewrite the loop as: + // for(a:b) c; ==> for(__iter=std::begin(b); __iter!=std::end(b); ++__iter) { a = *__iter; c; } + // where __iter is stored as a state variable + + if (WillContinue(stmt.body)) + { + StateVar container = state.FirstOrDefault(s => s.name == stmt.rangeExpression); + if (container == null) + { + throw new Error(stmt.FirstSourceLine, "container of range-based for with continuation must be a state variable"); + } + + var iter = getIteratorName(cx); + state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, name = iter, type = "decltype(std::begin(std::declval<" + container.type + ">()))", initializer = null }); + var equivalent = new ForStatement { + initExpression = iter + " = std::begin(" + stmt.rangeExpression + ")", + condExpression = iter + " != std::end(" + stmt.rangeExpression + ")", + nextExpression = "++" + iter, + FirstSourceLine = stmt.FirstSourceLine, + body = new CodeBlock + { + statements = new Statement[] { + new PlainOldCodeStatement { FirstSourceLine = stmt.FirstSourceLine, code = stmt.rangeDecl + " = *" + iter + ";" }, + stmt.body + } + } + }; + CompileStatement(equivalent, cx); + } + else + { + EmitNativeLoop(stmt.FirstSourceLine, "for( " + stmt.rangeDecl + " : " + stmt.rangeExpression + " )", stmt.body, cx); + } + } + + void CompileStatement(WhileStatement stmt, Context cx) + { + // Compile while (x) { y } as for(;x;) { y } + var equivalent = new ForStatement + { + condExpression = stmt.expression, + body = stmt.body, + FirstSourceLine = stmt.FirstSourceLine, + }; + + CompileStatement(equivalent, cx); + } + + void CompileStatement(LoopStatement stmt, Context cx) + { + // Compile loop { body } as for(;;;) { body } + var equivalent = new ForStatement + { + body = stmt.body, + FirstSourceLine = stmt.FirstSourceLine + }; + CompileStatement(equivalent, cx); + } + + // Writes out a loop in native C++ (with no continuation passing) + // Returns true if the loop is known to have no normal exit (is unreachable) + private bool EmitNativeLoop(int sourceLine, string head, Statement body, Context cx) + { + LineNumber(cx.target, sourceLine); + cx.target.WriteLine(head + " {"); + cx.target.Indent(+1); + var literalBreak = new LiteralBreak(); + Compile(AsCodeBlock(body), cx.LoopContext(cx.target, literalBreak, new LiteralContinue(), 0), true); + cx.target.Indent(-1); + cx.target.WriteLine("}"); + return !literalBreak.wasCalled; + } + + void CompileStatement(ChooseStatement stmt, Context cx) + { + int group = ++this.chooseGroups; + //string cbGroup = "ChooseGroup" + getFunction(cx.target.name,"W").name; // SOMEDAY + + var codeblock = stmt.body as CodeBlock; + if (codeblock == null) + throw new Error(stmt.FirstSourceLine, "'choose' must be followed by a compound statement."); + var choices = codeblock.statements + .OfType() + .Select( (ch,i) => new { + Stmt = ch, + Group = group, + Index = this.whenCount+i, + Body = getFunction(cx.target.name, "when", + new string[] { string.Format("{0} const& {2}{1}", ch.wait.result.type, ch.wait.result.name, ch.wait.resultIsState?"__":""), loopDepth }, + new string[] { string.Format("{0} && {2}{1}", ch.wait.result.type, ch.wait.result.name, ch.wait.resultIsState?"__":""), loopDepth } + ), + Future = string.Format("__when_expr_{0}", this.whenCount + i), + CallbackType = string.Format("{3}< {0}, {1}, {2} >", fullClassName, this.whenCount + i, ch.wait.result.type, ch.wait.isWaitNext ? "ActorSingleCallback" : "ActorCallback"), + CallbackTypeInStateClass = string.Format("{3}< {0}, {1}, {2} >", className, this.whenCount + i, ch.wait.result.type, ch.wait.isWaitNext ? "ActorSingleCallback" : "ActorCallback") + }) + .ToArray(); + this.whenCount += choices.Length; + if (choices.Length != codeblock.statements.Length) + throw new Error(codeblock.statements.First(x=>!(x is WhenStatement)).FirstSourceLine, "only 'when' statements are valid in an 'choose' block."); + + var exitFunc = getFunction("exitChoose", ""); + exitFunc.returnType = "void"; + exitFunc.WriteLine("if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", This); + foreach(var ch in choices) + exitFunc.WriteLine("{0}->{1}::remove();", This, ch.CallbackTypeInStateClass); + exitFunc.endIsUnreachable = true; + + //state.Add(new StateVar { SourceLine = stmt.FirstSourceLine, type = "CallbackGroup", name = cbGroup, callbackCatchFErr = cx.catchFErr }); + bool reachable = false; + foreach(var ch in choices) { + callbacks.Add(new CallbackVar + { + SourceLine = ch.Stmt.FirstSourceLine, + CallbackGroup = ch.Group, + type = ch.CallbackType + }); + var r = ch.Body; + if (ch.Stmt.wait.resultIsState) + { + Function overload = r.popOverload(); + CompileStatement(new StateDeclarationStatement + { + FirstSourceLine = ch.Stmt.FirstSourceLine, + decl = new VarDeclaration { + type = ch.Stmt.wait.result.type, + name = ch.Stmt.wait.result.name, + initializer = "__" + ch.Stmt.wait.result.name, + initializerConstructorSyntax = false + } + }, cx.WithTarget(r)); + if (overload != null) + { + overload.WriteLine("{0} = std::move(__{0});", ch.Stmt.wait.result.name); + r.setOverload(overload); + } + } + if (ch.Stmt.body != null) + { + r = Compile(AsCodeBlock(ch.Stmt.body), cx.WithTarget(r), true).target; + } + if (r != null) + { + reachable = true; + if (cx.next.formalParameters.Length == 1) + r.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); + else { + Function overload = r.popOverload(); + r.WriteLine("loopDepth = {0};", cx.next.call(ch.Stmt.wait.result.name, "loopDepth")); + if (overload != null) { + overload.WriteLine("loopDepth = {0};", cx.next.call(string.Format("std::move({0})", ch.Stmt.wait.result.name), "loopDepth")); + r.setOverload(overload); + } + } + } + + var cbFunc = new Function { + name = "callback_fire", + returnType = "void", + formalParameters = new string[] { + ch.CallbackTypeInStateClass + "*", + ch.Stmt.wait.result.type + " const& value" + }, + endIsUnreachable = true + }; + cbFunc.addOverload(ch.CallbackTypeInStateClass + "*", ch.Stmt.wait.result.type + " && value"); + functions.Add(string.Format("{0}#{1}", cbFunc.name, ch.Index), cbFunc); + cbFunc.Indent(codeIndent); + ProbeEnter(cbFunc, actor.name, ch.Index); + cbFunc.WriteLine("{0};", exitFunc.call()); + + Function _overload = cbFunc.popOverload(); + TryCatch(cx.WithTarget(cbFunc), cx.catchFErr, cx.tryLoopDepth, () => { + cbFunc.WriteLine("{0};", ch.Body.call("value", "0")); + }, false); + if (_overload != null) { + TryCatch(cx.WithTarget(_overload), cx.catchFErr, cx.tryLoopDepth, () => { + _overload.WriteLine("{0};", ch.Body.call("std::move(value)", "0")); + }, false); + cbFunc.setOverload(_overload); + } + ProbeExit(cbFunc, actor.name, ch.Index); + + var errFunc = new Function + { + name = "callback_error", + returnType = "void", + formalParameters = new string[] { + ch.CallbackTypeInStateClass + "*", + "Error err" + }, + endIsUnreachable = true + }; + functions.Add(string.Format("{0}#{1}", errFunc.name, ch.Index), errFunc); + errFunc.Indent(codeIndent); + ProbeEnter(errFunc, actor.name, ch.Index); + errFunc.WriteLine("{0};", exitFunc.call()); + TryCatch(cx.WithTarget(errFunc), cx.catchFErr, cx.tryLoopDepth, () => + { + errFunc.WriteLine("{0};", cx.catchFErr.call("err", "0")); + }, false); + ProbeExit(errFunc, actor.name, ch.Index); + } + + bool firstChoice = true; + foreach (var ch in choices) + { + string getFunc = ch.Stmt.wait.isWaitNext ? "pop" : "get"; + LineNumber(cx.target, ch.Stmt.wait.FirstSourceLine); + if (ch.Stmt.wait.isWaitNext) { + cx.target.WriteLine("auto {0} = {1};", ch.Future, ch.Stmt.wait.futureExpression); + cx.target.WriteLine("static_assert(std::is_same>::value || std::is_same>::value, \"invalid type\");", ch.Future, ch.Stmt.wait.result.type); + + } else { + cx.target.WriteLine("{2}<{3}> {0} = {1};", ch.Future, ch.Stmt.wait.futureExpression, "StrictFuture", ch.Stmt.wait.result.type); + } + + if (firstChoice) + { + // Do this check only after evaluating the expression for the first wait expression, so that expression cannot be short circuited by cancellation. + // So wait( expr() ) will always evaluate `expr()`, but choose { when ( wait(success( expr2() )) {} } need + // not evaluate `expr2()`. + firstChoice = false; + LineNumber(cx.target, stmt.FirstSourceLine); + if (actor.IsCancellable()) + cx.target.WriteLine("if ({1}->actor_wait_state < 0) return {0};", cx.catchFErr.call("actor_cancelled()", AdjustLoopDepth(cx.tryLoopDepth)), This); + } + + cx.target.WriteLine("if ({0}.isReady()) {{ if ({0}.isError()) return {2}; else return {1}; }};", ch.Future, ch.Body.call(ch.Future + "." + getFunc + "()", "loopDepth"), cx.catchFErr.call(ch.Future + ".getError()", AdjustLoopDepth(cx.tryLoopDepth))); + } + cx.target.WriteLine("{1}->actor_wait_state = {0};", group, This); + foreach (var ch in choices) + { + LineNumber(cx.target, ch.Stmt.wait.FirstSourceLine); + cx.target.WriteLine("{0}.addCallbackAndClear(static_cast<{1}*>({2}));", ch.Future, ch.CallbackTypeInStateClass, This); + } + cx.target.WriteLine("loopDepth = 0;");//cx.target.WriteLine("return 0;"); + + if (!reachable) cx.unreachable(); + } + void CompileStatement(BreakStatement stmt, Context cx) + { + if (cx.breakF == null) + throw new Error(stmt.FirstSourceLine, "break outside loop"); + if (cx.breakF is LiteralBreak) + cx.target.WriteLine("{0};", cx.breakF.call()); + else + { + cx.target.WriteLine("return {0}; // break", cx.breakF.call("loopDepth==0?0:loopDepth-1")); + } + cx.unreachable(); + } + void CompileStatement(ContinueStatement stmt, Context cx) + { + if (cx.continueF == null) + throw new Error(stmt.FirstSourceLine, "continue outside loop"); + if (cx.continueF is LiteralContinue) + cx.target.WriteLine("{0};", cx.continueF.call()); + else + cx.target.WriteLine("return {0}; // continue", cx.continueF.call("loopDepth")); + cx.unreachable(); + } + void CompileStatement(WaitStatement stmt, Context cx) + { + var equiv = new ChooseStatement + { + body = new CodeBlock + { + statements = new Statement[] { + new WhenStatement { + wait = stmt, + body = null, + FirstSourceLine = stmt.FirstSourceLine, + } + }, + FirstSourceLine = stmt.FirstSourceLine + }, + FirstSourceLine = stmt.FirstSourceLine + }; + if (!stmt.resultIsState) { + cx.next.formalParameters = new string[] { + string.Format("{0} const& {1}", stmt.result.type, stmt.result.name), + loopDepth }; + cx.next.addOverload( + string.Format("{0} && {1}", stmt.result.type, stmt.result.name), + loopDepth); + } + CompileStatement(equiv, cx); + } + void CompileStatement(CodeBlock stmt, Context cx) + { + cx.target.WriteLine("{"); + cx.target.Indent(+1); + var end = Compile(stmt, cx, true); + cx.target.Indent(-1); + cx.target.WriteLine("}"); + if (end.target == null) + cx.unreachable(); + else if (end.target != cx.target) + end.target.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); + } + void CompileStatement(ReturnStatement stmt, Context cx) + { + LineNumber(cx.target, stmt.FirstSourceLine); + if ((stmt.expression == "") != (actor.returnType == null)) + throw new Error(stmt.FirstSourceLine, "Return statement does not match actor declaration"); + if (actor.returnType != null) + { + if (stmt.expression == "Never()") + { + // `return Never();` destroys state immediately but never returns to the caller + cx.target.WriteLine("this->~{0}();", stateClassName); + cx.target.WriteLine("{0}->sendAndDelPromiseRef(Never());", This); + } + else + { + // Short circuit if there are no futures outstanding, but still evaluate the expression + // if it has side effects + cx.target.WriteLine("if (!{0}->SAV<{1}>::futures) {{ (void)({2}); this->~{3}(); {0}->destroy(); return 0; }}", This, actor.returnType, stmt.expression, stateClassName); + // Build the return value directly in SAV::value_storage + // If the expression is exactly the name of a state variable, std::move() it + if (state.Exists(s => s.name == stmt.expression)) + { + cx.target.WriteLine("new (&{0}->SAV< {1} >::value()) {1}(std::move({2})); // state_var_RVO", This, actor.returnType, stmt.expression); + } + else + { + cx.target.WriteLine("new (&{0}->SAV< {1} >::value()) {1}({2});", This, actor.returnType, stmt.expression); + } + // Destruct state + cx.target.WriteLine("this->~{0}();", stateClassName); + // Tell SAV to return the value we already constructed in value_storage + cx.target.WriteLine("{0}->finishSendAndDelPromiseRef();", This); + } + } else + cx.target.WriteLine("delete {0};", This); + cx.target.WriteLine("return 0;"); + cx.unreachable(); + } + void CompileStatement(IfStatement stmt, Context cx) + { + bool useContinuation = WillContinue(stmt.ifBody) || WillContinue(stmt.elseBody); + + LineNumber(cx.target, stmt.FirstSourceLine); + cx.target.WriteLine("if {1}({0})", stmt.expression, stmt.constexpr ? "constexpr " : ""); + cx.target.WriteLine("{"); + cx.target.Indent(+1); + Function ifTarget = Compile(AsCodeBlock(stmt.ifBody), cx, useContinuation).target; + if (useContinuation && ifTarget != null) + ifTarget.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); + cx.target.Indent(-1); + cx.target.WriteLine("}"); + Function elseTarget = null; + if (stmt.elseBody != null || useContinuation) + { + cx.target.WriteLine("else"); + cx.target.WriteLine("{"); + cx.target.Indent(+1); + elseTarget = cx.target; + if (stmt.elseBody != null) + { + elseTarget = Compile(AsCodeBlock(stmt.elseBody), cx, useContinuation).target; + } + if (useContinuation && elseTarget != null) + elseTarget.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); + cx.target.Indent(-1); + cx.target.WriteLine("}"); + } + if (ifTarget == null && stmt.elseBody != null && elseTarget == null) + cx.unreachable(); + else if (!cx.next.wasCalled && useContinuation) + throw new Exception("Internal error: IfStatement: next not called?"); + } + void CompileStatement(TryStatement stmt, Context cx) + { + bool reachable = false; + + if (stmt.catches.Count != 1) throw new Error(stmt.FirstSourceLine, "try statement must have exactly one catch clause"); + var c = stmt.catches[0]; + string catchErrorParameterName = ""; + if (c.expression != "...") { + string exp = c.expression.Replace(" ",""); + if (!exp.StartsWith("Error&")) + throw new Error(c.FirstSourceLine, "Only type 'Error' or '...' may be caught in an actor function"); + catchErrorParameterName = exp.Substring(6); + } + if (catchErrorParameterName == "") catchErrorParameterName = "__current_error"; + + var catchFErr = getFunction(cx.target.name, "Catch", "const Error& " + catchErrorParameterName, loopDepth0); + catchFErr.exceptionParameterIs = catchErrorParameterName; + var end = TryCatchCompile(AsCodeBlock(stmt.tryBody), cx.WithCatch(catchFErr)); + if (end.target != null) reachable = true; + + if (end.target!=null) + TryCatch(end, cx.catchFErr, cx.tryLoopDepth, () => + end.target.WriteLine("loopDepth = {0};", cx.next.call("loopDepth"))); + + // Now to write the catch function + TryCatch(cx.WithTarget(catchFErr), cx.catchFErr, cx.tryLoopDepth, () => + { + var cend = Compile(AsCodeBlock(c.body), cx.WithTarget(catchFErr), true); + if (cend.target != null) cend.target.WriteLine("loopDepth = {0};", cx.next.call("loopDepth")); + if (cend.target != null) reachable = true; + }); + + if (!reachable) cx.unreachable(); + } + void CompileStatement(ThrowStatement stmt, Context cx) + { + LineNumber(cx.target, stmt.FirstSourceLine); + + if (stmt.expression == "") + { + if (cx.target.exceptionParameterIs != null) + cx.target.WriteLine("return {0};", cx.catchFErr.call(cx.target.exceptionParameterIs, AdjustLoopDepth( cx.tryLoopDepth ))); + else + throw new Error(stmt.FirstSourceLine, "throw statement with no expression has no current exception in scope"); + } + else + cx.target.WriteLine("return {0};", cx.catchFErr.call(stmt.expression, AdjustLoopDepth( cx.tryLoopDepth ))); + cx.unreachable(); + } + void CompileStatement(Statement stmt, Context cx) + { + // Use reflection for double dispatch. SOMEDAY: Use a Dictionary> and expression trees to memoize + var method = typeof(ActorCompiler).GetMethod("CompileStatement", + System.Reflection.BindingFlags.NonPublic|System.Reflection.BindingFlags.Instance|System.Reflection.BindingFlags.ExactBinding, + null, new Type[] { stmt.GetType(), typeof(Context) }, null); + if (method == null) + throw new Error(stmt.FirstSourceLine, "Statement type {0} not supported yet.", stmt.GetType().Name); + try + { + method.Invoke(this, new object[] { stmt, cx }); + } + catch (System.Reflection.TargetInvocationException e) + { + if (!(e.InnerException is Error)) + Console.Error.WriteLine("\tHit error <{0}> for statement type {1} at line {2}\n\tStack Trace:\n{3}", + e.InnerException.Message, stmt.GetType().Name, stmt.FirstSourceLine, e.InnerException.StackTrace); + throw e.InnerException; + } + } + + // Compile returns a new context based on the one that is passed in, but (unlike CompileStatement) + // does not modify its parameter + // The target of the returned context is null if the end of the CodeBlock is unreachable (otherwise + // it is the target Function to which the end of the CodeBlock was written) + Context Compile(CodeBlock block, Context context, bool okToContinue=true) + { + var cx = context.Clone(); cx.next = null; + foreach (var stmt in block.statements) + { + if (cx.target == null) + { + throw new Error(stmt.FirstSourceLine, "Unreachable code."); + //Console.Error.WriteLine("\t(WARNING) Unreachable code at line {0}.", stmt.FirstSourceLine); + //break; + } + if (cx.next == null) + cx.next = getFunction(cx.target.name, "cont", loopDepth); + CompileStatement(stmt, cx); + if (cx.next.wasCalled) + { + if (cx.target == null) throw new Exception("Unreachable continuation called?"); + if (!okToContinue) throw new Exception("Unexpected continuation"); + cx.target = cx.next; + cx.next = null; + } + } + return cx; + } + + Dictionary functions = new Dictionary(); + + void WriteFunctions(TextWriter writer) + { + foreach (var func in functions.Values) + { + string body = func.BodyText; + if (body.Length != 0) + { + WriteFunction(writer, func, body); + } + if (func.overload != null) + { + string overloadBody = func.overload.BodyText; + if (overloadBody.Length != 0) + { + WriteFunction(writer, func.overload, overloadBody); + } + } + } + } + + private static void WriteFunction(TextWriter writer, Function func, string body) + { + writer.WriteLine(memberIndentStr + "{0}{1}({2}){3}", + func.returnType == "" ? "" : func.returnType + " ", + func.useByName(), + string.Join(",", func.formalParameters), + func.specifiers == "" ? "" : " " + func.specifiers); + if (func.returnType != "") + writer.WriteLine(memberIndentStr + "{"); + writer.WriteLine(body); + if (!func.endIsUnreachable) + writer.WriteLine(memberIndentStr + "\treturn loopDepth;"); + writer.WriteLine(memberIndentStr + "}"); + } + + Function getFunction(string baseName, string addName, string[] formalParameters, string[] overloadFormalParameters) + { + string proposedName; + if (addName == "cont" && baseName.Length>=5 && baseName.Substring(baseName.Length - 5, 4) == "cont") + proposedName = baseName.Substring(0, baseName.Length - 1); + else + proposedName = baseName + addName; + + int i = 0; + while (functions.ContainsKey(string.Format("{0}{1}", proposedName, ++i))) ; + + var f = new Function { + name = string.Format("{0}{1}", proposedName, i), + returnType = "int", + formalParameters = formalParameters + }; + if (overloadFormalParameters != null) { + f.addOverload(overloadFormalParameters); + } + f.Indent(codeIndent); + functions.Add(f.name, f); + return f; + } + + Function getFunction(string baseName, string addName, params string[] formalParameters) + { + return getFunction(baseName, addName, formalParameters, null); + } + + string[] ParameterList() + { + return actor.parameters.Select(p => + { + // SOMEDAY: pass small built in types by value + if (p.initializer != "") + return string.Format("{0} const& {1} = {2}", p.type, p.name, p.initializer); + else + return string.Format("{0} const& {1}", p.type, p.name); + }).ToArray(); + } + void WriteCancelFunc(TextWriter writer) + { + if (actor.IsCancellable()) + { + Function cancelFunc = new Function + { + name = "cancel", + returnType = "void", + formalParameters = new string[] {}, + endIsUnreachable = true, + publicName = true, + specifiers = "override" + }; + cancelFunc.Indent(codeIndent); + cancelFunc.WriteLine("auto wait_state = this->actor_wait_state;"); + cancelFunc.WriteLine("this->actor_wait_state = -1;"); + cancelFunc.WriteLine("switch (wait_state) {"); + int lastGroup = -1; + foreach (var cb in callbacks.OrderBy(cb => cb.CallbackGroup)) + if (cb.CallbackGroup != lastGroup) + { + lastGroup = cb.CallbackGroup; + cancelFunc.WriteLine("case {0}: this->a_callback_error(({1}*)0, actor_cancelled()); break;", cb.CallbackGroup, cb.type); + } + cancelFunc.WriteLine("}"); + WriteFunction(writer, cancelFunc, cancelFunc.BodyText); + } + } + + void WriteConstructor(Function body, TextWriter writer, string fullStateClassName) + { + Function constructor = new Function + { + name = className, + returnType = "", + formalParameters = ParameterList(), + endIsUnreachable = true, + publicName = true + }; + + // Initializes class member variables + constructor.Indent(codeIndent); + constructor.WriteLine( " : Actor<" + (actor.returnType == null ? "void" : actor.returnType) + ">()," ); + constructor.WriteLine( " {0}({1}),", fullStateClassName, string.Join(", ", actor.parameters.Select(p => p.name))); + constructor.WriteLine( " activeActorHelper(__actorIdentifier)"); + constructor.Indent(-1); + + constructor.WriteLine("{"); + constructor.Indent(+1); + + ProbeEnter(constructor, actor.name); + + constructor.WriteLine("#ifdef ENABLE_SAMPLING"); + constructor.WriteLine("this->lineage.setActorName(\"{0}\");", actor.name); + constructor.WriteLine("LineageScope _(&this->lineage);"); + // constructor.WriteLine("getCurrentLineage()->modify(&StackLineage::actorName) = \"{0}\"_sr;", actor.name); + constructor.WriteLine("#endif"); + + constructor.WriteLine("this->{0};", body.call()); + + ProbeExit(constructor, actor.name); + + WriteFunction(writer, constructor, constructor.BodyText); + } + + void WriteStateConstructor(TextWriter writer) + { + Function constructor = new Function + { + name = stateClassName, + returnType = "", + formalParameters = ParameterList(), + endIsUnreachable = true, + publicName = true + }; + constructor.Indent(codeIndent); + string ini = null; + int line = actor.SourceLine; + var initializers = state.AsEnumerable(); + foreach (var s in initializers) + if (s.initializer != null) + { + LineNumber(constructor, line); + if (ini != null) + { + constructor.WriteLine(ini + ","); + ini = " "; + } + else + { + ini = " : "; + } + + ini += string.Format("{0}({1})", s.name, s.initializer); + line = s.SourceLine; + } + LineNumber(constructor, line); + if (ini != null) + constructor.WriteLine(ini); + constructor.Indent(-1); + constructor.WriteLine("{"); + constructor.Indent(+1); + ProbeCreate(constructor, actor.name); + WriteFunction(writer, constructor, constructor.BodyText); + } + + void WriteStateDestructor(TextWriter writer) { + Function destructor = new Function + { + name = String.Format("~{0}", stateClassName), + returnType = "", + formalParameters = new string[0], + endIsUnreachable = true, + publicName = true, + }; + destructor.Indent(codeIndent); + destructor.Indent(-1); + destructor.WriteLine("{"); + destructor.Indent(+1); + ProbeDestroy(destructor, actor.name); + WriteFunction(writer, destructor, destructor.BodyText); + } + + IEnumerable Flatten(Statement stmt) + { + if (stmt == null) return new Statement[] { }; + var fl = new TypeSwitch>(stmt) + .Case(s => Flatten(s.body)) + .Case(s => Flatten(s.body)) + .Case(s => Flatten(s.body)) + .Case(s => Flatten(s.body)) + .Case(s => s.statements.SelectMany(t=>Flatten(t))) + .Case( s => Flatten(s.ifBody).Concat(Flatten(s.elseBody)) ) + .Case( s => Flatten(s.body) ) + .Case( s => Flatten(s.body) ) + .Case( s => Flatten(s.tryBody).Concat( s.catches.SelectMany(c=>Flatten(c.body)) ) ) + .Case(s => Enumerable.Empty()) + .Return(); + return new Statement[]{stmt}.Concat(fl); + } + + void FindState() + { + state = actor.parameters + .Select( + p=>new StateVar { SourceLine = actor.SourceLine, name=p.name, type=p.type, initializer=p.name, initializerConstructorSyntax=false } ) + .ToList(); + } + + // Generate an expression equivalent to max(0, loopDepth-subtract) for the given constant subtract + string AdjustLoopDepth(int subtract) + { + if (subtract == 0) + return "loopDepth"; + else + return string.Format("std::max(0, loopDepth - {0})", subtract); + } + } +} diff --git a/flow/actorcompiler/ActorCompiler.targets b/flow/actorcompiler/ActorCompiler.targets new file mode 100644 index 00000000000..16eaec6a8af --- /dev/null +++ b/flow/actorcompiler/ActorCompiler.targets @@ -0,0 +1,54 @@ + + + + + + + true + + + + + + + + _ActorCompiler + + + + + + + + + + $(BuildGenerateSourcesTargets); + ComputeACOutput; + + + + + + + + + + + + + + + + diff --git a/flow/actorcompiler/ActorCompiler.xml b/flow/actorcompiler/ActorCompiler.xml new file mode 100644 index 00000000000..19ed212ddcf --- /dev/null +++ b/flow/actorcompiler/ActorCompiler.xml @@ -0,0 +1,62 @@ + + + + + + + + + + Options + + + + + Command Line + + + + + + Compile generated file + + + The resulting file is a C++ module to be compiled, not a header file to be included + + + + + Actor Compiler Options + + + Actor Compiler Options + + + + + Additional C++ Options + + + Options passed to the C++ compiler processing the generated code + + + + + + + + + + \ No newline at end of file diff --git a/flow/actorcompiler/ActorParser.cs b/flow/actorcompiler/ActorParser.cs new file mode 100644 index 00000000000..7eb5cf53cb0 --- /dev/null +++ b/flow/actorcompiler/ActorParser.cs @@ -0,0 +1,1178 @@ +/* + * ActorParser.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace actorcompiler +{ + class Error : Exception + { + public int SourceLine { get; private set; } + public Error(int SourceLine, string format, params object[] args) + : base(string.Format(format,args)) + { + this.SourceLine = SourceLine; + } + }; + + class ErrorMessagePolicy + { + public bool DisableDiagnostics = false; + public void HandleActorWithoutWait(String sourceFile, Actor actor) + { + if (!DisableDiagnostics && !actor.isTestCase) + { + // TODO(atn34): Once cmake is the only build system we can make this an error instead of a warning. + Console.Error.WriteLine("{0}:{1}: warning: ACTOR {2} does not contain a wait() statement", sourceFile, actor.SourceLine, actor.name); + } + } + public bool ActorsNoDiscardByDefault() { + return !DisableDiagnostics; + } + } + + class Token + { + public string Value; + public int Position; + public int SourceLine; + public int BraceDepth; + public int ParenDepth; + public bool IsWhitespace { get { return Value == " " || Value == "\n" || Value == "\r" || Value == "\r\n" || Value == "\t" || Value.StartsWith("//") || Value.StartsWith("/*"); } } + public override string ToString() { return Value; } + public Token Assert(string error, Func pred) + { + if (!pred(this)) throw new Error(SourceLine, error); + return this; + } + public TokenRange GetMatchingRangeIn(TokenRange range) + { + Func pred; + int dir; + switch (Value) { + case "(": pred = t=> t.Value != ")" || t.ParenDepth != ParenDepth; dir = +1; break; + case ")": pred = t=> t.Value != "(" || t.ParenDepth != ParenDepth; dir = -1; break; + case "{": pred = t=> t.Value != "}" || t.BraceDepth != BraceDepth; dir = +1; break; + case "}": pred = t=> t.Value != "{" || t.BraceDepth != BraceDepth; dir = -1; break; + case "<": return + new TokenRange(range.GetAllTokens(), + Position+1, + AngleBracketParser.NotInsideAngleBrackets( + new TokenRange(range.GetAllTokens(), Position, range.End)) + .Skip(1) // skip the "<", which is considered "outside" + .First() // get the ">", which is likewise "outside" + .Position); + case "[": return + new TokenRange(range.GetAllTokens(), + Position+1, + BracketParser.NotInsideBrackets( + new TokenRange(range.GetAllTokens(), Position, range.End)) + .Skip(1) // skip the "[", which is considered "outside" + .First() // get the "]", which is likewise "outside" + .Position); + default: throw new NotSupportedException("Can't match this token!"); + } + TokenRange r; + if (dir == -1) + { + r = new TokenRange(range.GetAllTokens(), range.Begin, Position) + .RevTakeWhile(pred); + if (r.Begin == range.Begin) + throw new Error(SourceLine, "Syntax error: Unmatched " + Value); + } + else + { + r = new TokenRange(range.GetAllTokens(), Position+1, range.End) + .TakeWhile(pred); + if (r.End == range.End) + throw new Error(SourceLine, "Syntax error: Unmatched " + Value); + } + return r; + } + }; + + class TokenRange : IEnumerable + { + public TokenRange(Token[] tokens, int beginPos, int endPos) + { + if (beginPos > endPos) throw new InvalidOperationException("Invalid TokenRange"); + this.tokens = tokens; + this.beginPos = beginPos; + this.endPos = endPos; + } + + public bool IsEmpty { get { return beginPos==endPos; } } + public int Begin { get { return beginPos; } } + public int End { get { return endPos; } } + public Token First() { + if (beginPos == endPos) throw new InvalidOperationException("Empty TokenRange"); + return tokens[beginPos]; + } + public Token Last() { + if (beginPos == endPos) throw new InvalidOperationException("Empty TokenRange"); + return tokens[endPos - 1]; + } + public Token Last(Func pred) + { + for (int i = endPos - 1; i >= beginPos; i--) + if (pred(tokens[i])) + return tokens[i]; + throw new Exception("Matching token not found"); + } + public TokenRange Skip(int count) + { + return new TokenRange(tokens, beginPos + count, endPos); + } + public TokenRange Consume(string value) + { + First().Assert("Expected " + value, t => t.Value == value); + return Skip(1); + } + public TokenRange Consume(string error, Func pred) + { + First().Assert(error, pred); + return Skip(1); + } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return GetEnumerator(); } + public IEnumerator GetEnumerator() + { + for (int i = beginPos; i < endPos; i++) + yield return tokens[i]; + } + public TokenRange SkipWhile(Func pred) + { + for (int e = beginPos; e < endPos; e++) + if (!pred(tokens[e])) + return new TokenRange(tokens, e, endPos); + return new TokenRange(tokens, endPos, endPos); + } + public TokenRange TakeWhile(Func pred) + { + for (int e = beginPos; e < endPos; e++) + if (!pred(tokens[e])) + return new TokenRange(tokens, beginPos, e); + return new TokenRange(tokens, beginPos, endPos); + } + public TokenRange RevTakeWhile(Func pred) + { + for (int e = endPos-1; e >= beginPos; e--) + if (!pred(tokens[e])) + return new TokenRange(tokens, e+1, endPos); + return new TokenRange(tokens, beginPos, endPos); + } + public TokenRange RevSkipWhile(Func pred) + { + for (int e = endPos - 1; e >= beginPos; e--) + if (!pred(tokens[e])) + return new TokenRange(tokens, beginPos, e + 1); + return new TokenRange(tokens, beginPos, beginPos); + } + public Token[] GetAllTokens() { return tokens; } + + public int Length { + get { + return endPos - beginPos; + } + } + + Token[] tokens; + int beginPos; + int endPos; + }; + + static class BracketParser + { + public static IEnumerable NotInsideBrackets(IEnumerable tokens) + { + int BracketDepth = 0; + int? BasePD = null; + foreach (var tok in tokens) + { + if (BasePD == null) BasePD = tok.ParenDepth; + if (tok.ParenDepth == BasePD && tok.Value == "]") BracketDepth--; + if (BracketDepth == 0) + yield return tok; + if (tok.ParenDepth == BasePD && tok.Value == "[") BracketDepth++; + } + } + }; + static class AngleBracketParser + { + public static IEnumerable NotInsideAngleBrackets(IEnumerable tokens) + { + int AngleDepth = 0; + int? BasePD = null; + foreach (var tok in tokens) + { + if (BasePD == null) BasePD = tok.ParenDepth; + if (tok.ParenDepth == BasePD && tok.Value == ">") AngleDepth--; + if (AngleDepth == 0) + yield return tok; + if (tok.ParenDepth == BasePD && tok.Value == "<") AngleDepth++; + } + } + }; + + class ActorParser + { + public bool LineNumbersEnabled = true; + + Token[] tokens; + string sourceFile; + ErrorMessagePolicy errorMessagePolicy; + public bool generateProbes; + public Dictionary<(ulong, ulong), string> uidObjects { get; private set; } + + public ActorParser(string text, string sourceFile, ErrorMessagePolicy errorMessagePolicy, bool generateProbes) + { + this.sourceFile = sourceFile; + this.errorMessagePolicy = errorMessagePolicy; + this.generateProbes = generateProbes; + this.uidObjects = new Dictionary<(ulong, ulong), string>(); + tokens = Tokenize(text).Select(t=>new Token{ Value=t }).ToArray(); + CountParens(); + //if (sourceFile.EndsWith(".h")) LineNumbersEnabled = false; + //Console.WriteLine("{0} chars -> {1} tokens", text.Length, tokens.Length); + //showTokens(); + } + + class ClassContext { + public string name; + public int inBlocks; + } + + private bool ParseClassContext(TokenRange toks, out string name) + { + name = ""; + if (toks.Begin == toks.End) + { + return false; + } + + // http://nongnu.org/hcb/#attribute-specifier-seq + Token first; + while (true) + { + first = toks.First(NonWhitespace); + if (first.Value == "[") + { + var contents = first.GetMatchingRangeIn(toks); + toks = range(contents.End + 1, toks.End); + } + else if (first.Value == "alignas") + { + toks = range(first.Position + 1, toks.End); + first = toks.First(NonWhitespace); + first.Assert("Expected ( after alignas", t => t.Value == "("); + var contents = first.GetMatchingRangeIn(toks); + toks = range(contents.End + 1, toks.End); + } + else + { + break; + } + } + + // http://nongnu.org/hcb/#class-head-name + first = toks.First(NonWhitespace); + if (!identifierPattern.Match(first.Value).Success) { + return false; + } + while (true) { + first.Assert("Expected identifier", t=>identifierPattern.Match(t.Value).Success); + name += first.Value; + toks = range(first.Position + 1, toks.End); + if (toks.First(NonWhitespace).Value == "::") { + name += "::"; + toks = toks.SkipWhile(Whitespace).Skip(1); + } else { + break; + } + first = toks.First(NonWhitespace); + } + // http://nongnu.org/hcb/#class-virt-specifier-seq + toks = toks.SkipWhile(t => Whitespace(t) || t.Value == "final" || t.Value == "explicit"); + + first = toks.First(NonWhitespace); + if (first.Value == ":" || first.Value == "{") { + // At this point we've confirmed that this is a class. + return true; + } + return false; + } + + public void Write(System.IO.TextWriter writer, string destFileName) + { + writer.NewLine = "\n"; + writer.WriteLine("#define POST_ACTOR_COMPILER 1"); + int outLine = 1; + if (LineNumbersEnabled) + { + writer.WriteLine("#line {0} \"{1}\"", tokens[0].SourceLine, sourceFile); + outLine++; + } + int inBlocks = 0; + Stack classContextStack = new Stack(); + for(int i=0; i 0) + { + actor.enclosingClass = String.Join("::", classContextStack.Reverse().Select(t => t.name)); + } + var actorWriter = new System.IO.StringWriter(); + actorWriter.NewLine = "\n"; + var actorCompiler = new ActorCompiler(actor, sourceFile, inBlocks == 0, LineNumbersEnabled, generateProbes); + actorCompiler.Write(actorWriter); + actorCompiler.uidObjects.ToList().ForEach(x => this.uidObjects.TryAdd(x.Key, x.Value)); + + string[] actorLines = actorWriter.ToString().Split('\n'); + + bool hasLineNumber = false; + bool hadLineNumber = true; + foreach (var line in actorLines) + { + if (LineNumbersEnabled) + { + bool isLineNumber = line.Contains("#line"); + if (isLineNumber) hadLineNumber = true; + if (!isLineNumber && !hasLineNumber && hadLineNumber) + { + writer.WriteLine("\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} \"{1}\"", outLine + 1, destFileName); + outLine++; + hadLineNumber = false; + } + hasLineNumber = isLineNumber; + } + writer.WriteLine(line.TrimEnd('\n','\r')); + outLine++; + } + + i = end; + if (i != tokens.Length && LineNumbersEnabled) + { + writer.WriteLine("#line {0} \"{1}\"", tokens[i].SourceLine, sourceFile); + outLine++; + } + } + else if (tokens[i].Value == "DESCR") + { + int end; + var descr = ParseDescr(i, out end); + int lines; + new DescrCompiler(descr, tokens[i].BraceDepth).Write(writer, out lines); + i = end; + outLine += lines; + if (i != tokens.Length && LineNumbersEnabled) + { + writer.WriteLine("#line {0} \"{1}\"", tokens[i].SourceLine, sourceFile); + outLine++; + } + } + else if (tokens[i].Value == "class" || tokens[i].Value == "struct" || tokens[i].Value == "union") + { + writer.Write(tokens[i].Value); + string name; + if (ParseClassContext(range(i+1, tokens.Length), out name)) + { + classContextStack.Push(new ClassContext { name = name, inBlocks = inBlocks}); + } + } + else + { + if (tokens[i].Value == "{") + { + inBlocks++; + } + else if (tokens[i].Value == "}") + { + inBlocks--; + if (classContextStack.Count > 0 && classContextStack.Peek().inBlocks == inBlocks) + { + classContextStack.Pop(); + } + } + writer.Write(tokens[i].Value); + outLine += tokens[i].Value.Count(c => c == '\n'); + } + } + } + + IEnumerable SplitParameterList( TokenRange toks, string delimiter ) { + if (toks.Begin==toks.End) yield break; + while (true) { + Token comma = AngleBracketParser.NotInsideAngleBrackets( toks ) + .FirstOrDefault( t=> t.Value==delimiter && t.ParenDepth == toks.First().ParenDepth ); + if (comma == null) break; + yield return range(toks.Begin,comma.Position); + toks = range(comma.Position + 1, toks.End); + } + yield return toks; + } + + IEnumerable NormalizeWhitespace(IEnumerable tokens) + { + bool inWhitespace = false; + bool leading = true; + foreach (var tok in tokens) + { + if (!tok.IsWhitespace) + { + if (inWhitespace && !leading) yield return new Token { Value = " " }; + inWhitespace = false; + yield return tok; + leading = false; + } + else + { + inWhitespace = true; + } + } + } + + void ParseDeclaration(TokenRange tokens, + out Token name, + out TokenRange type, + out TokenRange initializer, + out bool constructorSyntax) + { + initializer = null; + TokenRange beforeInitializer = tokens; + constructorSyntax = false; + + Token equals = AngleBracketParser.NotInsideAngleBrackets(tokens) + .FirstOrDefault(t => t.Value == "=" && t.ParenDepth == tokens.First().ParenDepth); + if (equals != null) + { + // type name = initializer; + beforeInitializer = range(tokens.Begin,equals.Position); + initializer = range(equals.Position + 1, tokens.End); + } + else + { + Token paren = AngleBracketParser.NotInsideAngleBrackets(tokens) + .FirstOrDefault(t => t.Value == "("); + if (paren != null) + { + // type name(initializer); + constructorSyntax = true; + beforeInitializer = range(tokens.Begin, paren.Position); + initializer = + range(paren.Position + 1, tokens.End) + .TakeWhile(t => t.ParenDepth > paren.ParenDepth); + } else { + Token brace = AngleBracketParser.NotInsideAngleBrackets(tokens).FirstOrDefault(t => t.Value == "{"); + if (brace != null) { + // type name{initializer}; + throw new Error(brace.SourceLine, "Uniform initialization syntax is not currently supported for state variables (use '(' instead of '}}' ?)"); + } + } + } + name = beforeInitializer.Last(NonWhitespace); + if (beforeInitializer.Begin == name.Position) + throw new Error(beforeInitializer.First().SourceLine, "Declaration has no type."); + type = range(beforeInitializer.Begin, name.Position); + } + + VarDeclaration ParseVarDeclaration(TokenRange tokens) + { + Token name; + TokenRange type, initializer; + bool constructorSyntax; + ParseDeclaration( tokens, out name, out type, out initializer, out constructorSyntax ); + return new VarDeclaration + { + name = name.Value, + type = str(NormalizeWhitespace(type)), + initializer = initializer == null ? "" : str(NormalizeWhitespace(initializer)), + initializerConstructorSyntax = constructorSyntax + }; + } + + readonly Func Whitespace = (Token t) => t.IsWhitespace; + readonly Func NonWhitespace = (Token t) => !t.IsWhitespace; + + void ParseDescrHeading(Descr descr, TokenRange toks) + { + toks.First(NonWhitespace).Assert("non-struct DESCR!", t => t.Value == "struct"); + toks = toks.SkipWhile(Whitespace).Skip(1).SkipWhile(Whitespace); + + var colon = toks.FirstOrDefault(t => t.Value == ":"); + if (colon != null) + { + descr.superClassList = str(range(colon.Position + 1, toks.End)).Trim(); + toks = range(toks.Begin, colon.Position); + } + descr.name = str(toks).Trim(); + } + + void ParseTestCaseHeading(Actor actor, TokenRange toks) + { + actor.isStatic = true; + + // The parameter(s) to the TEST_CASE macro are opaque to the actor compiler + TokenRange paramRange = toks.Last(NonWhitespace) + .Assert("Unexpected tokens after test case parameter list.", + t => t.Value == ")" && t.ParenDepth == toks.First().ParenDepth) + .GetMatchingRangeIn(toks); + actor.testCaseParameters = str(paramRange); + + actor.name = "flowTestCase" + toks.First().SourceLine; + actor.parameters = new VarDeclaration[] { new VarDeclaration { + name = "params", + type = "UnitTestParameters", + initializer = "", + initializerConstructorSyntax = false + } + }; + actor.returnType = "Void"; + } + + void ParseActorHeading(Actor actor, TokenRange toks) + { + var template = toks.First(NonWhitespace); + if (template.Value == "template") + { + var templateParams = range(template.Position+1, toks.End) + .First(NonWhitespace) + .Assert("Invalid template declaration", t=>t.Value=="<") + .GetMatchingRangeIn(toks); + + actor.templateFormals = SplitParameterList(templateParams, ",") + .Select(p => ParseVarDeclaration(p)) //< SOMEDAY: ? + .ToArray(); + + toks = range(templateParams.End + 1, toks.End); + } + var attribute = toks.First(NonWhitespace); + while (attribute.Value == "[") + { + var attributeContents = attribute.GetMatchingRangeIn(toks); + + var asArray = attributeContents.ToArray(); + if (asArray.Length < 2 || asArray[0].Value != "[" || asArray[asArray.Length - 1].Value != "]") + { + throw new Error(actor.SourceLine, "Invalid attribute: Expected [[...]]"); + } + actor.attributes.Add("[" + str(NormalizeWhitespace(attributeContents)) + "]"); + toks = range(attributeContents.End + 1, toks.End); + + attribute = toks.First(NonWhitespace); + } + + var staticKeyword = toks.First(NonWhitespace); + if (staticKeyword.Value == "static") + { + actor.isStatic = true; + toks = range(staticKeyword.Position + 1, toks.End); + } + + var uncancellableKeyword = toks.First(NonWhitespace); + if (uncancellableKeyword.Value == "UNCANCELLABLE") + { + actor.SetUncancellable(); + toks = range(uncancellableKeyword.Position + 1, toks.End); + } + + // Find the parameter list + TokenRange paramRange = toks.Last(NonWhitespace) + .Assert("Unexpected tokens after actor parameter list.", + t => t.Value == ")" && t.ParenDepth == toks.First().ParenDepth) + .GetMatchingRangeIn(toks); + actor.parameters = SplitParameterList(paramRange, ",") + .Select(p => ParseVarDeclaration(p)) + .ToArray(); + + var name = range(toks.Begin,paramRange.Begin-1).Last(NonWhitespace); + actor.name = name.Value; + + // SOMEDAY: refactor? + var returnType = range(toks.First().Position + 1, name.Position).SkipWhile(Whitespace); + var retToken = returnType.First(); + if (retToken.Value == "Future") + { + var ofType = returnType.Skip(1).First(NonWhitespace).Assert("Expected <", tok => tok.Value == "<").GetMatchingRangeIn(returnType); + actor.returnType = str(NormalizeWhitespace(ofType)); + toks = range(ofType.End + 1, returnType.End); + } + else if (retToken.Value == "void"/* && !returnType.Skip(1).Any(NonWhitespace)*/) + { + actor.returnType = null; + toks = returnType.Skip(1); + } + else + throw new Error(actor.SourceLine, "Actor apparently does not return Future"); + + toks = toks.SkipWhile(Whitespace); + if (!toks.IsEmpty) + { + if (toks.Last().Value == "::") + { + actor.nameSpace = str(range(toks.Begin, toks.End - 1)); + } + else + { + Console.WriteLine("Tokens: '{0}' {1} '{2}'", str(toks), toks.Count(), toks.Last().Value); + throw new Error(actor.SourceLine, "Unrecognized tokens preceding parameter list in actor declaration"); + } + } + if (errorMessagePolicy.ActorsNoDiscardByDefault() && !actor.attributes.Contains("[[flow_allow_discard]]")) { + if (actor.IsCancellable()) + { + actor.attributes.Add("[[nodiscard]]"); + } + } + HashSet knownFlowAttributes = new HashSet(); + knownFlowAttributes.Add("[[flow_allow_discard]]"); + foreach (var flowAttribute in actor.attributes.Where(a => a.StartsWith("[[flow_"))) { + if (!knownFlowAttributes.Contains(flowAttribute)) { + throw new Error(actor.SourceLine, "Unknown flow attribute {0}", flowAttribute); + } + } + actor.attributes = actor.attributes.Where(a => !a.StartsWith("[[flow_")).ToList(); + } + + LoopStatement ParseLoopStatement(TokenRange toks) + { + return new LoopStatement { + body = ParseCompoundStatement( toks.Consume("loop") ) + }; + } + + ChooseStatement ParseChooseStatement(TokenRange toks) + { + return new ChooseStatement + { + body = ParseCompoundStatement(toks.Consume("choose")) + }; + } + + WhenStatement ParseWhenStatement(TokenRange toks) + { + var expr = toks.Consume("when") + .SkipWhile(Whitespace) + .First() + .Assert("Expected (", t => t.Value == "(") + .GetMatchingRangeIn(toks) + .SkipWhile(Whitespace); + + return new WhenStatement { + wait = ParseWaitStatement(expr), + body = ParseCompoundStatement(range(expr.End+1, toks.End)) + }; + } + + StateDeclarationStatement ParseStateDeclaration(TokenRange toks) + { + toks = toks.Consume("state").RevSkipWhile(t => t.Value == ";"); + return new StateDeclarationStatement { + decl = ParseVarDeclaration(toks) + }; + } + + ReturnStatement ParseReturnStatement(TokenRange toks) + { + toks = toks.Consume("return").RevSkipWhile(t => t.Value == ";"); + return new ReturnStatement + { + expression = str(NormalizeWhitespace(toks)) + }; + } + + ThrowStatement ParseThrowStatement(TokenRange toks) + { + toks = toks.Consume("throw").RevSkipWhile(t => t.Value == ";"); + return new ThrowStatement + { + expression = str(NormalizeWhitespace(toks)) + }; + } + + WaitStatement ParseWaitStatement(TokenRange toks) + { + WaitStatement ws = new WaitStatement(); + ws.FirstSourceLine = toks.First().SourceLine; + if (toks.First().Value == "state") + { + ws.resultIsState = true; + toks = toks.Consume("state"); + } + TokenRange initializer; + if (toks.First().Value == "wait" || toks.First().Value == "waitNext") + { + initializer = toks.RevSkipWhile(t=>t.Value==";"); + ws.result = new VarDeclaration { + name = "_", + type = "Void", + initializer = "", + initializerConstructorSyntax = false + }; + } else { + Token name; + TokenRange type; + bool constructorSyntax; + ParseDeclaration( toks.RevSkipWhile(t=>t.Value==";"), out name, out type, out initializer, out constructorSyntax ); + + string typestring = str(NormalizeWhitespace(type)); + if (typestring == "Void") { + throw new Error(ws.FirstSourceLine, "Assigning the result of a Void wait is not allowed. Just use a standalone wait statement."); + } + + ws.result = new VarDeclaration + { + name = name.Value, + type = str(NormalizeWhitespace(type)), + initializer = "", + initializerConstructorSyntax = false + }; + } + + if (initializer == null) throw new Error(ws.FirstSourceLine, "Wait statement must be a declaration or standalone statement"); + + var waitParams = initializer + .SkipWhile(Whitespace).Consume("Statement contains a wait, but is not a valid wait statement or a supported compound statement.1", + t=> { + if (t.Value=="wait") return true; + if (t.Value=="waitNext") { ws.isWaitNext = true; return true; } + return false; + }) + .SkipWhile(Whitespace).First().Assert("Expected (", t => t.Value == "(") + .GetMatchingRangeIn(initializer); + if (!range(waitParams.End, initializer.End).Consume(")").All(Whitespace)) { + throw new Error(toks.First().SourceLine, "Statement contains a wait, but is not a valid wait statement or a supported compound statement.2"); + } + + ws.futureExpression = str(NormalizeWhitespace(waitParams)); + return ws; + } + + WhileStatement ParseWhileStatement(TokenRange toks) + { + var expr = toks.Consume("while") + .First(NonWhitespace) + .Assert("Expected (", t => t.Value == "(") + .GetMatchingRangeIn(toks); + return new WhileStatement + { + expression = str(NormalizeWhitespace(expr)), + body = ParseCompoundStatement(range(expr.End + 1, toks.End)) + }; + } + + Statement ParseForStatement(TokenRange toks) + { + var head = + toks.Consume("for") + .First(NonWhitespace) + .Assert("Expected (", t => t.Value == "(") + .GetMatchingRangeIn(toks); + + Token[] delim = + head.Where( + t => t.ParenDepth == head.First().ParenDepth && + t.BraceDepth == head.First().BraceDepth && + t.Value==";" + ).ToArray(); + if (delim.Length == 2) + { + var init = range(head.Begin, delim[0].Position); + var cond = range(delim[0].Position + 1, delim[1].Position); + var next = range(delim[1].Position + 1, head.End); + var body = range(head.End + 1, toks.End); + + return new ForStatement + { + initExpression = str(NormalizeWhitespace(init)), + condExpression = str(NormalizeWhitespace(cond)), + nextExpression = str(NormalizeWhitespace(next)), + body = ParseCompoundStatement(body) + }; + } + + delim = + head.Where( + t => t.ParenDepth == head.First().ParenDepth && + t.BraceDepth == head.First().BraceDepth && + t.Value == ":" + ).ToArray(); + if (delim.Length != 1) + { + throw new Error(head.First().SourceLine, "for statement must be 3-arg style or c++11 2-arg style"); + } + + return new RangeForStatement + { + // The container over which to iterate + rangeExpression = str(NormalizeWhitespace(range(delim[0].Position + 1, head.End).SkipWhile(Whitespace))), + // Type and name of the variable assigned in each iteration + rangeDecl = str(NormalizeWhitespace(range(head.Begin, delim[0].Position - 1).SkipWhile(Whitespace))), + // The body of the for loop + body = ParseCompoundStatement(range(head.End + 1, toks.End)) + }; + } + + Statement ParseIfStatement(TokenRange toks) + { + toks = toks.Consume("if"); + toks = toks.SkipWhile(Whitespace); + bool constexpr = toks.First().Value == "constexpr"; + if(constexpr) { + toks = toks.Consume("constexpr").SkipWhile(Whitespace); + } + + var expr = toks.First(NonWhitespace) + .Assert("Expected (", t => t.Value == "(") + .GetMatchingRangeIn(toks); + return new IfStatement { + expression = str(NormalizeWhitespace(expr)), + constexpr = constexpr, + ifBody = ParseCompoundStatement(range(expr.End+1, toks.End)) + // elseBody will be filled in later if necessary by ParseElseStatement + }; + } + void ParseElseStatement(TokenRange toks, Statement prevStatement) + { + var ifStatement = prevStatement as IfStatement; + while (ifStatement != null && ifStatement.elseBody != null) + ifStatement = ifStatement.elseBody as IfStatement; + if (ifStatement == null) + throw new Error(toks.First().SourceLine, "else without matching if"); + ifStatement.elseBody = ParseCompoundStatement(toks.Consume("else")); + } + + Statement ParseTryStatement(TokenRange toks) + { + return new TryStatement + { + tryBody = ParseCompoundStatement(toks.Consume("try")), + catches = new List() // will be filled in later by ParseCatchStatement + }; + } + void ParseCatchStatement(TokenRange toks, Statement prevStatement) + { + var tryStatement = prevStatement as TryStatement; + if (tryStatement == null) + throw new Error(toks.First().SourceLine, "catch without matching try"); + var expr = toks.Consume("catch") + .First(NonWhitespace) + .Assert("Expected (", t => t.Value == "(") + .GetMatchingRangeIn(toks); + tryStatement.catches.Add( + new TryStatement.Catch + { + expression = str(NormalizeWhitespace(expr)), + body = ParseCompoundStatement(range(expr.End + 1, toks.End)), + FirstSourceLine = expr.First().SourceLine + }); + } + + static readonly HashSet IllegalKeywords = new HashSet { "goto", "do", "finally", "__if_exists", "__if_not_exists" }; + + void ParseDeclaration(TokenRange toks, List declarations) + { + Declaration dec = new Declaration(); + + Token delim = toks.First(t => t.Value == ";"); + var nameRange = range(toks.Begin, delim.Position).RevSkipWhile(Whitespace).RevTakeWhile(NonWhitespace); + var typeRange = range(toks.Begin, nameRange.Begin); + var commentRange = range(delim.Position + 1, toks.End); + + dec.name = str(nameRange).Trim(); + dec.type = str(typeRange).Trim(); + dec.comment = str(commentRange).Trim().TrimStart('/'); + + declarations.Add(dec); + } + + void ParseStatement(TokenRange toks, List statements) + { + toks = toks.SkipWhile(Whitespace); + + Action Add = stmt => + { + stmt.FirstSourceLine = toks.First().SourceLine; + statements.Add(stmt); + }; + + switch (toks.First().Value) + { + case "loop": Add(ParseLoopStatement(toks)); break; + case "while": Add(ParseWhileStatement(toks)); break; + case "for": Add(ParseForStatement(toks)); break; + case "break": Add(new BreakStatement()); break; + case "continue": Add(new ContinueStatement()); break; + case "return": Add(ParseReturnStatement(toks)); break; + case "{": Add(ParseCompoundStatement(toks)); break; + case "if": Add(ParseIfStatement(toks)); break; + case "else": ParseElseStatement(toks, statements[statements.Count - 1]); break; + case "choose": Add(ParseChooseStatement(toks)); break; + case "when": Add(ParseWhenStatement(toks)); break; + case "try": Add(ParseTryStatement(toks)); break; + case "catch": ParseCatchStatement(toks, statements[statements.Count - 1]); break; + case "throw": Add(ParseThrowStatement(toks)); break; + default: + if (IllegalKeywords.Contains(toks.First().Value)) + throw new Error(toks.First().SourceLine, "Statement '{0}' not supported in actors.", toks.First().Value); + if (toks.Any(t => t.Value == "wait" || t.Value == "waitNext")) + Add(ParseWaitStatement(toks)); + else if (toks.First().Value == "state") + Add(ParseStateDeclaration(toks)); + else if (toks.First().Value == "switch" && toks.Any(t => t.Value == "return")) + throw new Error(toks.First().SourceLine, "Unsupported compound statement containing return."); + else if (toks.First().Value.StartsWith("#")) + throw new Error(toks.First().SourceLine, "Found \"{0}\". Preprocessor directives are not supported within ACTORs", toks.First().Value); + else if (toks.RevSkipWhile(t => t.Value == ";").Any(NonWhitespace)) + Add(new PlainOldCodeStatement + { + code = str(NormalizeWhitespace(toks.RevSkipWhile(t => t.Value == ";"))) + ";" + }); + break; + }; + } + + Statement ParseCompoundStatement(TokenRange toks) + { + var first = toks.First(NonWhitespace); + if (first.Value == "{") { + var inBraces = first.GetMatchingRangeIn(toks); + if (!range(inBraces.End, toks.End).Consume("}").All(Whitespace)) + throw new Error(inBraces.Last().SourceLine, "Unexpected tokens after compound statement"); + return ParseCodeBlock(inBraces); + } else { + List statements = new List(); + ParseStatement( toks.Skip(1), statements ); + return statements[0]; + } + } + + List ParseDescrCodeBlock(TokenRange toks) + { + List declarations = new List(); + while (true) + { + Token delim = toks.FirstOrDefault(t => t.Value == ";"); + if (delim == null) + break; + + int pos = delim.Position + 1; + var potentialComment = range(pos, toks.End).SkipWhile(t => t.Value == "\t" || t.Value == " "); + if (!potentialComment.IsEmpty && potentialComment.First().Value.StartsWith("//")) + { + pos = potentialComment.First().Position + 1; + } + + ParseDeclaration(range(toks.Begin, pos), declarations); + + toks = range(pos, toks.End); + } + if (!toks.All(Whitespace)) + throw new Error(toks.First(NonWhitespace).SourceLine, "Trailing unterminated statement in code block"); + return declarations; + } + + CodeBlock ParseCodeBlock(TokenRange toks) + { + List statements = new List(); + while (true) + { + Token delim = toks + .FirstOrDefault( + t=> t.ParenDepth == toks.First().ParenDepth && + t.BraceDepth == toks.First().BraceDepth && + (t.Value==";" || t.Value == "}") + ); + if (delim == null) + break; + ParseStatement(range(toks.Begin, delim.Position + 1), statements); + toks = range(delim.Position + 1, toks.End); + } + if (!toks.All(Whitespace)) + throw new Error(toks.First(NonWhitespace).SourceLine, "Trailing unterminated statement in code block"); + return new CodeBlock { statements = statements.ToArray() }; + } + + TokenRange range(int beginPos, int endPos) + { + return new TokenRange(tokens, beginPos, endPos); + } + + Descr ParseDescr(int pos, out int end) + { + var descr = new Descr(); + var toks = range(pos + 1, tokens.Length); + var heading = toks.TakeWhile(t => t.Value != "{"); + var body = range(heading.End + 1, tokens.Length) + .TakeWhile(t => t.BraceDepth > toks.First().BraceDepth || t.Value == ";" ); //assumes no whitespace between the last "}" and the ";" + + ParseDescrHeading(descr, heading); + descr.body = ParseDescrCodeBlock(body); + + end = body.End + 1; + return descr; + } + + Actor ParseActor( int pos, out int end ) { + var actor = new Actor(); + var head_token = tokens[pos]; + actor.SourceLine = head_token.SourceLine; + + var toks = range(pos+1, tokens.Length); + var heading = toks.TakeWhile(t => t.Value != "{"); + var toSemicolon = toks.TakeWhile(t => t.Value != ";"); + actor.isForwardDeclaration = toSemicolon.Length < heading.Length; + if (actor.isForwardDeclaration) { + heading = toSemicolon; + if (head_token.Value == "ACTOR" || head_token.Value == "SWIFT_ACTOR") { + ParseActorHeading(actor, heading); + } else { + head_token.Assert("ACTOR expected!", t => false); + } + end = heading.End + 1; + } else { + var body = range(heading.End+1, tokens.Length) + .TakeWhile(t => t.BraceDepth > toks.First().BraceDepth); + + if (head_token.Value == "ACTOR" || head_token.Value == "SWIFT_ACTOR") + { + ParseActorHeading(actor, heading); + } + else if (head_token.Value == "TEST_CASE") { + ParseTestCaseHeading(actor, heading); + actor.isTestCase = true; + } + else + head_token.Assert("ACTOR or TEST_CASE expected!", t => false); + + actor.body = ParseCodeBlock(body); + + if (!actor.body.containsWait()) + this.errorMessagePolicy.HandleActorWithoutWait(sourceFile, actor); + + end = body.End + 1; + } + return actor; + } + + string str(IEnumerable tokens) + { + return string.Join("", tokens.Select(x => x.Value).ToArray()); + } + string str(int begin, int end) + { + return str(range(begin,end)); + } + + void CountParens() + { + int BraceDepth = 0, ParenDepth = 0, LineCount = 1; + Token lastParen = null, lastBrace = null; + for (int i = 0; i < tokens.Length; i++) + { + switch (tokens[i].Value) + { + case "}": BraceDepth--; break; + case ")": ParenDepth--; break; + case "\r\n": LineCount++; break; + case "\n": LineCount++; break; + } + if (BraceDepth < 0) throw new Error(LineCount, "Mismatched braces"); + if (ParenDepth < 0) throw new Error(LineCount, "Mismatched parenthesis"); + tokens[i].Position = i; + tokens[i].SourceLine = LineCount; + tokens[i].BraceDepth = BraceDepth; + tokens[i].ParenDepth = ParenDepth; + if (tokens[i].Value.StartsWith("/*")) LineCount += tokens[i].Value.Count(c=>c=='\n'); + switch (tokens[i].Value) + { + case "{": BraceDepth++; if (BraceDepth==1) lastBrace = tokens[i]; break; + case "(": ParenDepth++; if (ParenDepth==1) lastParen = tokens[i]; break; + } + } + if (BraceDepth != 0) throw new Error(lastBrace.SourceLine, "Unmatched brace"); + if (ParenDepth != 0) throw new Error(lastParen.SourceLine, "Unmatched parenthesis"); + } + + void showTokens() + { + foreach (var t in tokens) + { + if (t.Value == "\r\n") + Console.WriteLine(); + else if (t.Value.Length == 1) + Console.Write(t.Value); + else + Console.Write("|{0}|", t.Value); + } + } + + readonly Regex identifierPattern = new Regex(@"\G[a-zA-Z_][a-zA-Z_0-9]*", RegexOptions.Singleline); + + readonly Regex[] tokenExpressions = (new string[] { + @"\{", + @"\}", + @"\(", + @"\)", + @"\[", + @"\]", + @"//[^\n]*", + @"/[*]([*][^/]|[^*])*[*]/", + @"'(\\.|[^\'\n])*'", //< SOMEDAY: Not fully restrictive + @"""(\\.|[^\""\n])*""", + @"[a-zA-Z_][a-zA-Z_0-9]*", + @"\r\n", + @"\n", + @"::", + @":", + @"#[a-z]*", // Recognize preprocessor directives so that we can reject them + @".", + }).Select( x=>new Regex(@"\G"+x, RegexOptions.Singleline) ).ToArray(); + + IEnumerable Tokenize(string text) + { + int pos = 0; + while (pos < text.Length) + { + bool ok = false; + foreach (var re in tokenExpressions) + { + var m = re.Match(text, pos); + if (m.Success) + { + yield return m.Value; + pos += m.Value.Length; + ok = true; + break; + } + } + if (!ok) + throw new Exception( String.Format("Can't tokenize! {0}", pos)); + } + } + } +} diff --git a/flow/actorcompiler/ParseTree.cs b/flow/actorcompiler/ParseTree.cs new file mode 100644 index 00000000000..766d56b304c --- /dev/null +++ b/flow/actorcompiler/ParseTree.cs @@ -0,0 +1,252 @@ +/* + * ParseTree.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace actorcompiler +{ + class VarDeclaration + { + public string type; + public string name; + public string initializer; + public bool initializerConstructorSyntax; + }; + + abstract class Statement + { + public int FirstSourceLine; + public virtual bool containsWait() + { + return false; + } + }; + class PlainOldCodeStatement : Statement + { + public string code; + public override string ToString() + { + return code; + } + }; + class StateDeclarationStatement : Statement + { + public VarDeclaration decl; + public override string ToString() + { + if (decl.initializerConstructorSyntax) + return string.Format("State {0} {1}({2});", decl.type, decl.name, decl.initializer); + else + return string.Format("State {0} {1} = {2};", decl.type, decl.name, decl.initializer); + } + }; + class WhileStatement : Statement + { + public string expression; + public Statement body; + public override bool containsWait() + { + return body.containsWait(); + } + }; + class ForStatement : Statement + { + public string initExpression = ""; + public string condExpression = ""; + public string nextExpression = ""; + public Statement body; + public override bool containsWait() + { + return body.containsWait(); + } + }; + class RangeForStatement : Statement + { + public string rangeExpression; + public string rangeDecl; + public Statement body; + public override bool containsWait() + { + return body.containsWait(); + } + }; + class LoopStatement : Statement + { + public Statement body; + public override string ToString() + { + return "Loop " + body.ToString(); + } + public override bool containsWait() + { + return body.containsWait(); + } + }; + class BreakStatement : Statement + { + }; + class ContinueStatement : Statement + { + }; + class IfStatement : Statement + { + public string expression; + public bool constexpr; + public Statement ifBody; + public Statement elseBody; // might be null + public override bool containsWait() + { + return ifBody.containsWait() || (elseBody != null && elseBody.containsWait()); + } + }; + class ReturnStatement : Statement + { + public string expression; + public override string ToString() + { + return "Return " + expression; + } + }; + class WaitStatement : Statement + { + public VarDeclaration result; + public string futureExpression; + public bool resultIsState; + public bool isWaitNext; + public override string ToString() + { + return string.Format("Wait {0} {1} <- {2} ({3})", result.type, result.name, futureExpression, resultIsState ? "state" : "local"); + } + public override bool containsWait() + { + return true; + } + }; + class ChooseStatement : Statement + { + public Statement body; + public override string ToString() + { + return "Choose " + body.ToString(); + } + public override bool containsWait() + { + return body.containsWait(); + } + }; + class WhenStatement : Statement + { + public WaitStatement wait; + public Statement body; + public override string ToString() + { + return string.Format("When ({0}) {1}", wait, body); + } + public override bool containsWait() + { + return true; + } + }; + class TryStatement : Statement + { + public struct Catch + { + public string expression; + public Statement body; + public int FirstSourceLine; + }; + + public Statement tryBody; + public List catches; + public override bool containsWait() + { + if (tryBody.containsWait()) + return true; + foreach (Catch c in catches) + if (c.body.containsWait()) + return true; + return false; + } + }; + class ThrowStatement : Statement + { + public string expression; + }; + + class CodeBlock : Statement + { + public Statement[] statements; + public override string ToString() + { + return string.Join("\n", + new string[] { "CodeBlock" } + .Concat(statements.Select(s => s.ToString())) + .Concat(new string[] { "EndCodeBlock" }) + .ToArray()); + } + public override bool containsWait() + { + foreach (Statement s in statements) + if (s.containsWait()) + return true; + return false; + } + }; + + + class Declaration + { + public string type; + public string name; + public string comment; + }; + + class Actor + { + public List attributes = new List(); + public string returnType; + public string name; + public string enclosingClass = null; + public VarDeclaration[] parameters; + public VarDeclaration[] templateFormals; //< null if not a template + public CodeBlock body; + public int SourceLine; + public bool isStatic = false; + private bool isUncancellable; + public string testCaseParameters = null; + public string nameSpace = null; + public bool isForwardDeclaration = false; + public bool isTestCase = false; + + public bool IsCancellable() { return returnType != null && !isUncancellable; } + public void SetUncancellable() { isUncancellable = true; } + }; + + class Descr + { + public string name; + public string superClassList; + public List body; + }; +}; diff --git a/flow/actorcompiler/Program.cs b/flow/actorcompiler/Program.cs new file mode 100644 index 00000000000..742dd21a24f --- /dev/null +++ b/flow/actorcompiler/Program.cs @@ -0,0 +1,104 @@ +/* + * Program.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.IO; + +namespace actorcompiler +{ + class Program + { + private static void OverwriteByMove(string target, string temporaryFile) { + if (File.Exists(target)) + { + File.SetAttributes(target, FileAttributes.Normal); + File.Delete(target); + } + File.Move(temporaryFile, target); + File.SetAttributes(target, FileAttributes.ReadOnly); + } + + public static int Main(string[] args) + { + bool generateProbes = false; + if (args.Length < 2) + { + Console.WriteLine("Usage:"); + Console.WriteLine(" actorcompiler [--disable-diagnostics] [--generate-probes]"); + return 100; + } + Console.WriteLine("actorcompiler {0}", string.Join(" ", args)); + string input = args[0], output = args[1], outputtmp = args[1] + ".tmp", outputUid = args[1] + ".uid"; + ErrorMessagePolicy errorMessagePolicy = new ErrorMessagePolicy(); + foreach (var arg in args) { + if (arg.StartsWith("--")) { + if (arg.Equals("--disable-diagnostics")) { + errorMessagePolicy.DisableDiagnostics = true; + } else if (arg.Equals("--generate-probes")) { + generateProbes = true; + } + } + } + try + { + var inputData = File.ReadAllText(input); + var parser = new ActorParser(inputData, input.Replace('\\', '/'), errorMessagePolicy, generateProbes); + + using (var outputStream = new StreamWriter(outputtmp)) { + parser.Write(outputStream, output.Replace('\\', '/')); + } + OverwriteByMove(output, outputtmp); + + using (var outputStream = new StreamWriter(outputtmp)) { + foreach(var entry in parser.uidObjects) { + outputStream.WriteLine("{0}|{1}|{2}", entry.Key.Item1, entry.Key.Item2, entry.Value); + } + } + OverwriteByMove(outputUid, outputtmp); + + return 0; + } + catch (actorcompiler.Error e) + { + Console.Error.WriteLine("{0}({1}): error FAC1000: {2}", input, e.SourceLine, e.Message); + if (File.Exists(outputtmp)) + File.Delete(outputtmp); + if (File.Exists(output)) + { + File.SetAttributes(output, FileAttributes.Normal); + File.Delete(output); + } + return 1; + } + catch (Exception e) + { + Console.Error.WriteLine("{0}({1}): error FAC2000: Internal {2}", input, 1, e.ToString()); + if (File.Exists(outputtmp)) + File.Delete(outputtmp); + if (File.Exists(output)) + { + File.SetAttributes(output, FileAttributes.Normal); + File.Delete(output); + } + return 3; + } + } + } +} diff --git a/flow/actorcompiler/Properties/AssemblyInfo.cs b/flow/actorcompiler/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..bec6889f4b5 --- /dev/null +++ b/flow/actorcompiler/Properties/AssemblyInfo.cs @@ -0,0 +1,56 @@ +/* + * AssemblyInfo.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("actorcompiler")] +[assembly: AssemblyDescription("Compile Flow code to C++")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Apple Inc")] +[assembly: AssemblyProduct("actorcompiler")] +[assembly: AssemblyCopyright("Copyright (c) 2013-2025 Apple Inc.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("fd8d34c8-46ca-43de-bfb3-b437e46943da")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/flow/actorcompiler/__main__.py b/flow/actorcompiler/__main__.py deleted file mode 100644 index 0effc52441c..00000000000 --- a/flow/actorcompiler/__main__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Allow running the actorcompiler as a module: - python3 -m flow.actorcompiler input.actor.cpp output.g.cpp -""" - -import sys -from .main import main - -if __name__ == "__main__": - sys.exit(main()) diff --git a/flow/actorcompiler/actor_parser.py b/flow/actorcompiler/actor_parser.py deleted file mode 100644 index f45977eeaa9..00000000000 --- a/flow/actorcompiler/actor_parser.py +++ /dev/null @@ -1,1134 +0,0 @@ -from __future__ import annotations - -import io -import re -import sys -from dataclasses import dataclass -from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple - -from . import ActorCompilerError -from .actor_compiler import ActorCompiler, DescrCompiler - -from .parse_tree import ( - Actor, - BreakStatement, - ChooseStatement, - CodeBlock, - ContinueStatement, - Declaration, - Descr, - ForStatement, - IfStatement, - LoopStatement, - PlainOldCodeStatement, - RangeForStatement, - ReturnStatement, - StateDeclarationStatement, - Statement, - ThrowStatement, - TryStatement, - VarDeclaration, - WaitStatement, - WhenStatement, - WhileStatement, -) - - -class ErrorMessagePolicy: - def __init__(self) -> None: - self.DisableDiagnostics = False - - def HandleActorWithoutWait(self, sourceFile: str, actor: Actor) -> None: - if not self.DisableDiagnostics and not actor.isTestCase: - print( - f"{sourceFile}:{actor.SourceLine}: warning: ACTOR {actor.name} does not contain a wait() statement", - file=sys.stderr, - ) - - def ActorsNoDiscardByDefault(self) -> bool: - return not self.DisableDiagnostics - - -@dataclass -class Token: - Value: str - Position: int = 0 - SourceLine: int = 0 - BraceDepth: int = 0 - ParenDepth: int = 0 - - @property - def IsWhitespace(self) -> bool: - return ( - self.Value in (" ", "\n", "\r", "\r\n", "\t") - or self.Value.startswith("//") - or self.Value.startswith("/*") - ) - - def Assert(self, error: str, pred) -> "Token": - if not pred(self): - raise ActorCompilerError(self.SourceLine, error) - return self - - def GetMatchingRangeIn(self, token_range: "TokenRange") -> "TokenRange": - if self.Value == "<": - sub = TokenRange(token_range.tokens, self.Position, token_range.End) - gen = AngleBracketParser.NotInsideAngleBrackets(sub) - next(gen, None) # skip the "<" - closing = next(gen, None) - if closing is None: - raise ActorCompilerError(self.SourceLine, "Syntax error: Unmatched <") - return TokenRange(token_range.tokens, self.Position + 1, closing.Position) - if self.Value == "[": - sub = TokenRange(token_range.tokens, self.Position, token_range.End) - gen = BracketParser.NotInsideBrackets(sub) - next(gen, None) # skip the "[" - closing = next(gen, None) - if closing is None: - raise ActorCompilerError(self.SourceLine, "Syntax error: Unmatched [") - return TokenRange(token_range.tokens, self.Position + 1, closing.Position) - - pairs = {"(": ")", ")": "(", "{": "}", "}": "{"} - if self.Value not in pairs: - raise RuntimeError("Can't match this token") - pred = ( - (lambda t: t.Value != ")" or t.ParenDepth != self.ParenDepth) - if self.Value == "(" - else ( - (lambda t: t.Value != "(" or t.ParenDepth != self.ParenDepth) - if self.Value == ")" - else ( - (lambda t: t.Value != "}" or t.BraceDepth != self.BraceDepth) - if self.Value == "{" - else (lambda t: t.Value != "{" or t.BraceDepth != self.BraceDepth) - ) - ) - ) - direction = 1 if self.Value in ("(", "{") else -1 - if direction == -1: - rng = token_range.Range(token_range.Begin, self.Position).RevTakeWhile(pred) - if rng.Begin == token_range.Begin: - raise ActorCompilerError( - self.SourceLine, f"Syntax error: Unmatched {self.Value}" - ) - return rng - rng = token_range.Range(self.Position + 1, token_range.End).TakeWhile(pred) - if rng.End == token_range.End: - raise ActorCompilerError( - self.SourceLine, f"Syntax error: Unmatched {self.Value}" - ) - return rng - - -class TokenRange: - def __init__(self, tokens: List[Token], begin: int, end: int) -> None: - if begin > end: - raise RuntimeError("Invalid TokenRange") - self.tokens = tokens - self.beginPos = begin - self.endPos = end - - @property - def Begin(self) -> int: - return self.beginPos - - @property - def End(self) -> int: - return self.endPos - - def __iter__(self) -> Iterator[Token]: - for i in range(self.beginPos, self.endPos): - yield self.tokens[i] - - def First(self, predicate=None) -> Token: - if self.beginPos == self.endPos: - raise RuntimeError("Empty TokenRange") - if predicate is None: - return self.tokens[self.beginPos] - for t in self: - if predicate(t): - return t - raise RuntimeError("Matching token not found") - - def Last(self, predicate=None) -> Token: - if self.beginPos == self.endPos: - raise RuntimeError("Empty TokenRange") - if predicate is None: - return self.tokens[self.endPos - 1] - for i in range(self.endPos - 1, self.beginPos - 1, -1): - if predicate(self.tokens[i]): - return self.tokens[i] - raise RuntimeError("Matching token not found") - - def Skip(self, count: int) -> "TokenRange": - return TokenRange(self.tokens, self.beginPos + count, self.endPos) - - def Consume(self, value_or_error, predicate=None) -> "TokenRange": - if predicate is None: - self.First().Assert( - f"Expected {value_or_error}", lambda t: t.Value == value_or_error - ) - else: - self.First().Assert(value_or_error, predicate) - return self.Skip(1) - - def SkipWhile(self, predicate) -> "TokenRange": - e = self.beginPos - while e < self.endPos and predicate(self.tokens[e]): - e += 1 - return TokenRange(self.tokens, e, self.endPos) - - def TakeWhile(self, predicate) -> "TokenRange": - e = self.beginPos - while e < self.endPos and predicate(self.tokens[e]): - e += 1 - return TokenRange(self.tokens, self.beginPos, e) - - def RevTakeWhile(self, predicate) -> "TokenRange": - e = self.endPos - 1 - while e >= self.beginPos and predicate(self.tokens[e]): - e -= 1 - return TokenRange(self.tokens, e + 1, self.endPos) - - def RevSkipWhile(self, predicate) -> "TokenRange": - e = self.endPos - 1 - while e >= self.beginPos and predicate(self.tokens[e]): - e -= 1 - return TokenRange(self.tokens, self.beginPos, e + 1) - - def Range(self, begin: int, end: int) -> "TokenRange": - return TokenRange(self.tokens, begin, end) - - def IsEmpty(self) -> bool: - return self.beginPos == self.endPos - - def Length(self) -> int: - return self.endPos - self.beginPos - - def All(self, predicate) -> bool: - return all(predicate(t) for t in self) - - def Any(self, predicate) -> bool: - return any(predicate(t) for t in self) - - -class BracketParser: - @staticmethod - def NotInsideBrackets(tokens: Iterable[Token]) -> Iterator[Token]: - bracket_depth = 0 - base_pd = None - for tok in tokens: - if base_pd is None: - base_pd = tok.ParenDepth - if tok.ParenDepth == base_pd and tok.Value == "]": - bracket_depth -= 1 - if bracket_depth == 0: - yield tok - if tok.ParenDepth == base_pd and tok.Value == "[": - bracket_depth += 1 - - -class AngleBracketParser: - @staticmethod - def NotInsideAngleBrackets(tokens: Iterable[Token]) -> Iterator[Token]: - angle_depth = 0 - base_pd = None - for tok in tokens: - if base_pd is None: - base_pd = tok.ParenDepth - if tok.ParenDepth == base_pd and tok.Value == ">": - angle_depth -= 1 - if angle_depth == 0: - yield tok - if tok.ParenDepth == base_pd and tok.Value == "<": - angle_depth += 1 - - -class ActorParser: - tokenExpressions = [ - r"\{", - r"\}", - r"\(", - r"\)", - r"\[", - r"\]", - r"//[^\n]*", - r"/[*]([*][^/]|[^*])*[*]/", - r"'(\\.|[^\'\n])*'", - r'"(\\.|[^"\n])*"', - r"[a-zA-Z_][a-zA-Z_0-9]*", - r"\r\n", - r"\n", - r"::", - r":", - r"#[a-z]*", - r".", - ] - - identifierPattern = re.compile(r"^[a-zA-Z_][a-zA-Z_0-9]*$") - - def __init__( - self, - text: str, - sourceFile: str, - errorMessagePolicy: ErrorMessagePolicy, - generateProbes: bool, - ) -> None: - self.sourceFile = sourceFile - self.errorMessagePolicy = errorMessagePolicy - self.generateProbes = generateProbes - self.tokens = [Token(Value=t) for t in self.Tokenize(text)] - self.LineNumbersEnabled = True - self.uidObjects: Dict[Tuple[int, int], str] = {} - self.TokenArray = self.tokens - self.CountParens() - - def Tokenize(self, text: str) -> List[str]: - regexes = [re.compile(pattern, re.S) for pattern in self.tokenExpressions] - pos = 0 - tokens: List[str] = [] - while pos < len(text): - for regex in regexes: - m = regex.match(text, pos) - if m: - tokens.append(m.group(0)) - pos += len(m.group(0)) - break - else: - raise RuntimeError(f"Can't tokenize! {pos}") - return tokens - - def CountParens(self) -> None: - brace_depth = 0 - paren_depth = 0 - line_count = 1 - last_paren = None - last_brace = None - for i, token in enumerate(self.tokens): - value = token.Value - if value == "}": - brace_depth -= 1 - elif value == ")": - paren_depth -= 1 - elif value == "\r\n": - line_count += 1 - elif value == "\n": - line_count += 1 - - if brace_depth < 0: - raise ActorCompilerError(line_count, "Mismatched braces") - if paren_depth < 0: - raise ActorCompilerError(line_count, "Mismatched parenthesis") - - token.Position = i - token.SourceLine = line_count - token.BraceDepth = brace_depth - token.ParenDepth = paren_depth - - if value.startswith("/*"): - line_count += value.count("\n") - if value == "{": - brace_depth += 1 - if brace_depth == 1: - last_brace = token - elif value == "(": - paren_depth += 1 - if paren_depth == 1: - last_paren = token - if brace_depth != 0: - raise ActorCompilerError( - last_brace.SourceLine if last_brace else line_count, "Unmatched brace" - ) - if paren_depth != 0: - raise ActorCompilerError( - last_paren.SourceLine if last_paren else line_count, - "Unmatched parenthesis", - ) - - def Write(self, writer: io.TextIOBase, destFileName: str) -> None: - ActorCompiler.usedClassNames.clear() - writer.write("#define POST_ACTOR_COMPILER 1\n") - outLine = 1 - if self.LineNumbersEnabled: - writer.write(f'#line {self.tokens[0].SourceLine} "{self.sourceFile}"\n') - outLine += 1 - inBlocks = 0 - classContextStack: List[Tuple[str, int]] = [] - i = 0 - while i < len(self.tokens): - tok = self.tokens[i] - if tok.SourceLine == 0: - raise RuntimeError("Invalid source line (0)") - if tok.Value in ("ACTOR", "SWIFT_ACTOR", "TEST_CASE"): - actor = self.ParseActor(i) - end = self._parse_end - if classContextStack: - actor.enclosingClass = "::".join( - name for name, _ in classContextStack - ) - actor_writer = io.StringIO() - actorCompiler = ActorCompiler( - actor, - self.sourceFile, - inBlocks == 0, - self.LineNumbersEnabled, - self.generateProbes, - ) - actorCompiler.Write(actor_writer) - for key, value in actorCompiler.uidObjects.items(): - self.uidObjects.setdefault(key, value) - actor_lines = actor_writer.getvalue().split("\n") - hasLineNumber = False - hadLineNumber = True - for line in actor_lines: - if self.LineNumbersEnabled: - isLine = "#line" in line - if isLine: - hadLineNumber = True - if not isLine and not hasLineNumber and hadLineNumber: - writer.write( - '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( - outLine + 1, destFileName - ) - ) - outLine += 1 - hadLineNumber = False - hasLineNumber = isLine - writer.write(line.rstrip("\n\r") + "\n") - outLine += 1 - i = end - if i < len(self.tokens) and self.LineNumbersEnabled: - writer.write( - f'#line {self.tokens[i].SourceLine} "{self.sourceFile}"\n' - ) - outLine += 1 - elif tok.Value == "DESCR": - descr, end = self.ParseDescr(i) - descr_writer = io.StringIO() - lines = DescrCompiler(descr, tok.BraceDepth).write(descr_writer) - writer.write(descr_writer.getvalue()) - outLine += lines - i = end - if i < len(self.tokens) and self.LineNumbersEnabled: - writer.write( - f'#line {self.tokens[i].SourceLine} "{self.sourceFile}"\n' - ) - outLine += 1 - elif tok.Value in ("class", "struct", "union"): - writer.write(tok.Value) - success, name = self.ParseClassContext( - self.range(i + 1, len(self.tokens)) - ) - if success: - classContextStack.append((name, inBlocks)) - else: - if tok.Value == "{": - inBlocks += 1 - elif tok.Value == "}": - inBlocks -= 1 - if classContextStack and classContextStack[-1][1] == inBlocks: - classContextStack.pop() - writer.write(tok.Value) - outLine += tok.Value.count("\n") - i += 1 - - # Parsing helpers - - def range(self, begin: int, end: int) -> TokenRange: - return TokenRange(self.tokens, begin, end) - - def ParseActor(self, pos: int) -> Actor: - token = self.tokens[pos] - actor = Actor() - actor.SourceLine = token.SourceLine - toks = self.range(pos + 1, len(self.tokens)) - heading = toks.TakeWhile(lambda t: t.Value != "{") - toSemicolon = toks.TakeWhile(lambda t: t.Value != ";") - actor.isForwardDeclaration = toSemicolon.Length() < heading.Length() - if actor.isForwardDeclaration: - heading = toSemicolon - if token.Value in ("ACTOR", "SWIFT_ACTOR"): - self.ParseActorHeading(actor, heading) - else: - token.Assert("ACTOR expected!", lambda _: False) - self._parse_end = heading.End + 1 - else: - body = self.range(heading.End + 1, len(self.tokens)).TakeWhile( - lambda t: t.BraceDepth > toks.First().BraceDepth - ) - if token.Value in ("ACTOR", "SWIFT_ACTOR"): - self.ParseActorHeading(actor, heading) - elif token.Value == "TEST_CASE": - self.ParseTestCaseHeading(actor, heading) - actor.isTestCase = True - else: - token.Assert("ACTOR or TEST_CASE expected!", lambda _: False) - actor.body = self.ParseCodeBlock(body) - if not actor.body.containsWait(): - self.errorMessagePolicy.HandleActorWithoutWait(self.sourceFile, actor) - self._parse_end = body.End + 1 - return actor - - def ParseDescr(self, pos: int) -> Tuple[Descr, int]: - descr = Descr() - toks = self.range(pos + 1, len(self.tokens)) - heading = toks.TakeWhile(lambda t: t.Value != "{") - body = self.range(heading.End + 1, len(self.tokens)).TakeWhile( - lambda t: t.BraceDepth > toks.First().BraceDepth or t.Value == ";" - ) - self.ParseDescrHeading(descr, heading) - descr.body = self.ParseDescrCodeBlock(body) - end = body.End + 1 - return descr, end - - def ParseDescrHeading(self, descr: Descr, toks: TokenRange) -> None: - nonWhitespace = lambda t: not t.IsWhitespace - toks.First(nonWhitespace).Assert( - "non-struct DESCR!", lambda t: t.Value == "struct" - ) - toks = ( - toks.SkipWhile(lambda t: t.IsWhitespace) - .Skip(1) - .SkipWhile(lambda t: t.IsWhitespace) - ) - colon = next((t for t in toks if t.Value == ":"), None) - if colon: - descr.superClassList = self.str( - self.range(colon.Position + 1, toks.End) - ).strip() - toks = self.range(toks.Begin, colon.Position) - descr.name = self.str(toks).strip() - - def ParseDescrCodeBlock(self, toks: TokenRange) -> List[Declaration]: - declarations: List[Declaration] = [] - while True: - delim = next((t for t in toks if t.Value == ";"), None) - if delim is None: - break - pos = delim.Position + 1 - potential_comment = self.range(pos, toks.End).SkipWhile( - lambda t: t.Value in ("\t", " ") - ) - if ( - not potential_comment.IsEmpty() - and potential_comment.First().Value.startswith("//") - ): - pos = potential_comment.First().Position + 1 - self.ParseDeclarationRange(self.range(toks.Begin, pos), declarations) - toks = self.range(pos, toks.End) - if not toks.All(lambda t: t.IsWhitespace): - raise ActorCompilerError( - toks.First(lambda t: not t.IsWhitespace).SourceLine, - "Trailing unterminated statement in code block", - ) - return declarations - - def ParseDeclarationRange( - self, toks: TokenRange, declarations: List[Declaration] - ) -> None: - delim = toks.First(lambda t: t.Value == ";") - nameRange = ( - self.range(toks.Begin, delim.Position) - .RevSkipWhile(lambda t: t.IsWhitespace) - .RevTakeWhile(lambda t: not t.IsWhitespace) - ) - typeRange = self.range(toks.Begin, nameRange.Begin) - commentRange = self.range(delim.Position + 1, toks.End) - declarations.append( - Declaration( - name=self.str(nameRange).strip(), - type=self.str(typeRange).strip(), - comment=self.str(commentRange).strip().lstrip("/"), - ) - ) - - def ParseTestCaseHeading(self, actor: Actor, toks: TokenRange) -> None: - actor.isStatic = True - nonWhitespace = lambda t: not t.IsWhitespace - paramRange = ( - toks.Last(nonWhitespace) - .Assert( - "Unexpected tokens after test case parameter list.", - lambda t: t.Value == ")" and t.ParenDepth == toks.First().ParenDepth, - ) - .GetMatchingRangeIn(toks) - ) - actor.testCaseParameters = self.str(paramRange) - actor.name = f"flowTestCase{toks.First().SourceLine}" - actor.parameters = [ - VarDeclaration( - name="params", - type="UnitTestParameters", - initializer="", - initializerConstructorSyntax=False, - ) - ] - actor.returnType = "Void" - - def ParseActorHeading(self, actor: Actor, toks: TokenRange) -> None: - nonWhitespace = lambda t: not t.IsWhitespace - template = toks.First(nonWhitespace) - if template.Value == "template": - templateParams = ( - self.range(template.Position + 1, toks.End) - .First(nonWhitespace) - .Assert("Invalid template declaration", lambda t: t.Value == "<") - .GetMatchingRangeIn(toks) - ) - actor.templateFormals = [ - self.ParseVarDeclaration(p) - for p in self.SplitParameterList(templateParams, ",") - ] - toks = self.range(templateParams.End + 1, toks.End) - attribute = toks.First(nonWhitespace) - while attribute.Value == "[": - contents = attribute.GetMatchingRangeIn(toks) - as_array = list(contents) - if ( - len(as_array) < 2 - or as_array[0].Value != "[" - or as_array[-1].Value != "]" - ): - raise ActorCompilerError( - actor.SourceLine, "Invalid attribute: Expected [[...]]" - ) - actor.attributes.append( - "[" + self.str(self.NormalizeWhitespace(contents)) + "]" - ) - toks = self.range(contents.End + 1, toks.End) - attribute = toks.First(nonWhitespace) - static_keyword = toks.First(nonWhitespace) - if static_keyword.Value == "static": - actor.isStatic = True - toks = self.range(static_keyword.Position + 1, toks.End) - uncancellable = toks.First(nonWhitespace) - if uncancellable.Value == "UNCANCELLABLE": - actor.SetUncancellable() - toks = self.range(uncancellable.Position + 1, toks.End) - paramRange = ( - toks.Last(nonWhitespace) - .Assert( - "Unexpected tokens after actor parameter list.", - lambda t: t.Value == ")" and t.ParenDepth == toks.First().ParenDepth, - ) - .GetMatchingRangeIn(toks) - ) - actor.parameters = [ - self.ParseVarDeclaration(p) - for p in self.SplitParameterList(paramRange, ",") - ] - nameToken = self.range(toks.Begin, paramRange.Begin - 1).Last(nonWhitespace) - actor.name = nameToken.Value - return_range = self.range( - toks.First().Position + 1, nameToken.Position - ).SkipWhile(lambda t: t.IsWhitespace) - retToken = return_range.First() - if retToken.Value == "Future": - ofType = ( - return_range.Skip(1) - .First(nonWhitespace) - .Assert("Expected <", lambda tok: tok.Value == "<") - .GetMatchingRangeIn(return_range) - ) - actor.returnType = self.str(self.NormalizeWhitespace(ofType)) - toks = self.range(ofType.End + 1, return_range.End) - elif retToken.Value == "void": - actor.returnType = None - toks = return_range.Skip(1) - else: - raise ActorCompilerError( - actor.SourceLine, "Actor apparently does not return Future" - ) - toks = toks.SkipWhile(lambda t: t.IsWhitespace) - if not toks.IsEmpty(): - if toks.Last().Value == "::": - actor.nameSpace = self.str(self.range(toks.Begin, toks.End - 1)) - else: - raise ActorCompilerError( - actor.SourceLine, - "Unrecognized tokens preceding parameter list in actor declaration", - ) - if ( - self.errorMessagePolicy.ActorsNoDiscardByDefault() - and "[[flow_allow_discard]]" not in actor.attributes - ): - if actor.IsCancellable(): - actor.attributes.append("[[nodiscard]]") - known_flow_attributes = {"[[flow_allow_discard]]"} - for flow_attribute in [a for a in actor.attributes if a.startswith("[[flow_")]: - if flow_attribute not in known_flow_attributes: - raise ActorCompilerError( - actor.SourceLine, f"Unknown flow attribute {flow_attribute}" - ) - actor.attributes = [a for a in actor.attributes if not a.startswith("[[flow_")] - - def ParseVarDeclaration(self, tokens: TokenRange) -> VarDeclaration: - name, typeRange, initializer, constructorSyntax = self.ParseDeclaration(tokens) - return VarDeclaration( - name=name.Value, - type=self.str(self.NormalizeWhitespace(typeRange)), - initializer=( - "" - if initializer is None - else self.str(self.NormalizeWhitespace(initializer)) - ), - initializerConstructorSyntax=constructorSyntax, - ) - - def ParseDeclaration(self, tokens: TokenRange): - nonWhitespace = lambda t: not t.IsWhitespace - initializer = None - beforeInitializer = tokens - constructorSyntax = False - equals = next( - ( - t - for t in AngleBracketParser.NotInsideAngleBrackets(tokens) - if t.Value == "=" and t.ParenDepth == tokens.First().ParenDepth - ), - None, - ) - if equals: - beforeInitializer = self.range(tokens.Begin, equals.Position) - initializer = self.range(equals.Position + 1, tokens.End) - else: - paren = next( - ( - t - for t in AngleBracketParser.NotInsideAngleBrackets(tokens) - if t.Value == "(" - ), - None, - ) - if paren: - constructorSyntax = True - beforeInitializer = self.range(tokens.Begin, paren.Position) - initializer = self.range(paren.Position + 1, tokens.End).TakeWhile( - lambda t, p=paren.ParenDepth: t.ParenDepth > p - ) - else: - brace = next( - ( - t - for t in AngleBracketParser.NotInsideAngleBrackets(tokens) - if t.Value == "{" - ), - None, - ) - if brace: - raise ActorCompilerError( - brace.SourceLine, - "Uniform initialization syntax is not currently supported for state variables (use '(' instead of '}' ?)", - ) - name = beforeInitializer.Last(nonWhitespace) - if beforeInitializer.Begin == name.Position: - raise ActorCompilerError( - beforeInitializer.First().SourceLine, "Declaration has no type." - ) - typeRange = self.range(beforeInitializer.Begin, name.Position) - return name, typeRange, initializer, constructorSyntax - - def NormalizeWhitespace(self, tokens: Iterable[Token]) -> Iterable[Token]: - inWhitespace = False - leading = True - for tok in tokens: - if not tok.IsWhitespace: - if inWhitespace and not leading: - yield Token(Value=" ") - inWhitespace = False - yield tok - leading = False - else: - inWhitespace = True - - def SplitParameterList( - self, toks: TokenRange, delimiter: str - ) -> Iterable[TokenRange]: - if toks.Begin == toks.End: - return [] - ranges: List[TokenRange] = [] - while True: - comma = next( - ( - t - for t in AngleBracketParser.NotInsideAngleBrackets(toks) - if t.Value == delimiter and t.ParenDepth == toks.First().ParenDepth - ), - None, - ) - if comma is None: - break - ranges.append(self.range(toks.Begin, comma.Position)) - toks = self.range(comma.Position + 1, toks.End) - ranges.append(toks) - return ranges - - def ParseLoopStatement(self, toks: TokenRange) -> LoopStatement: - return LoopStatement(body=self.ParseCompoundStatement(toks.Consume("loop"))) - - def ParseChooseStatement(self, toks: TokenRange) -> ChooseStatement: - return ChooseStatement(body=self.ParseCompoundStatement(toks.Consume("choose"))) - - def ParseWhenStatement(self, toks: TokenRange) -> WhenStatement: - expr = ( - toks.Consume("when") - .SkipWhile(lambda t: t.IsWhitespace) - .First() - .Assert("Expected (", lambda t: t.Value == "(") - .GetMatchingRangeIn(toks) - .SkipWhile(lambda t: t.IsWhitespace) - ) - return WhenStatement( - wait=self.ParseWaitStatement(expr), - body=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), - ) - - def ParseStateDeclaration(self, toks: TokenRange) -> StateDeclarationStatement: - toks = toks.Consume("state").RevSkipWhile(lambda t: t.Value == ";") - return StateDeclarationStatement(decl=self.ParseVarDeclaration(toks)) - - def ParseReturnStatement(self, toks: TokenRange) -> ReturnStatement: - toks = toks.Consume("return").RevSkipWhile(lambda t: t.Value == ";") - return ReturnStatement(expression=self.str(self.NormalizeWhitespace(toks))) - - def ParseThrowStatement(self, toks: TokenRange) -> ThrowStatement: - toks = toks.Consume("throw").RevSkipWhile(lambda t: t.Value == ";") - return ThrowStatement(expression=self.str(self.NormalizeWhitespace(toks))) - - def ParseWaitStatement(self, toks: TokenRange) -> WaitStatement: - ws = WaitStatement() - ws.FirstSourceLine = toks.First().SourceLine - if toks.First().Value == "state": - ws.resultIsState = True - toks = toks.Consume("state") - initializer = None - if toks.First().Value in ("wait", "waitNext"): - initializer = toks.RevSkipWhile(lambda t: t.Value == ";") - ws.result = VarDeclaration( - name="_", - type="Void", - initializer="", - initializerConstructorSyntax=False, - ) - else: - name, typeRange, initializer, constructorSyntax = self.ParseDeclaration( - toks.RevSkipWhile(lambda t: t.Value == ";") - ) - type_str = self.str(self.NormalizeWhitespace(typeRange)) - if type_str == "Void": - raise ActorCompilerError( - ws.FirstSourceLine, - "Assigning the result of a Void wait is not allowed. Just use a standalone wait statement.", - ) - ws.result = VarDeclaration( - name=name.Value, - type=self.str(self.NormalizeWhitespace(typeRange)), - initializer="", - initializerConstructorSyntax=False, - ) - if initializer is None: - raise ActorCompilerError( - ws.FirstSourceLine, - "Wait statement must be a declaration or standalone statement", - ) - waitParams = ( - initializer.SkipWhile(lambda t: t.IsWhitespace) - .Consume( - "Statement contains a wait, but is not a valid wait statement or a supported compound statement.1", - lambda t: True if t.Value in ("wait", "waitNext") else False, - ) - .SkipWhile(lambda t: t.IsWhitespace) - .First() - .Assert("Expected (", lambda t: t.Value == "(") - .GetMatchingRangeIn(initializer) - ) - if ( - not self.range(waitParams.End, initializer.End) - .Consume(")") - .All(lambda t: t.IsWhitespace) - ): - raise ActorCompilerError( - toks.First().SourceLine, - "Statement contains a wait, but is not a valid wait statement or a supported compound statement.2", - ) - ws.futureExpression = self.str(self.NormalizeWhitespace(waitParams)) - ws.isWaitNext = "waitNext" in [t.Value for t in initializer] - return ws - - def ParseWhileStatement(self, toks: TokenRange) -> WhileStatement: - expr = ( - toks.Consume("while") - .First(lambda t: not t.IsWhitespace) - .Assert("Expected (", lambda t: t.Value == "(") - .GetMatchingRangeIn(toks) - ) - return WhileStatement( - expression=self.str(self.NormalizeWhitespace(expr)), - body=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), - ) - - def ParseForStatement(self, toks: TokenRange) -> Statement: - head = ( - toks.Consume("for") - .First(lambda t: not t.IsWhitespace) - .Assert("Expected (", lambda t: t.Value == "(") - .GetMatchingRangeIn(toks) - ) - delim = [ - t - for t in head - if t.ParenDepth == head.First().ParenDepth - and t.BraceDepth == head.First().BraceDepth - and t.Value == ";" - ] - if len(delim) == 2: - init = self.range(head.Begin, delim[0].Position) - cond = self.range(delim[0].Position + 1, delim[1].Position) - next_expr = self.range(delim[1].Position + 1, head.End) - body = self.range(head.End + 1, toks.End) - return ForStatement( - initExpression=self.str(self.NormalizeWhitespace(init)), - condExpression=self.str(self.NormalizeWhitespace(cond)), - nextExpression=self.str(self.NormalizeWhitespace(next_expr)), - body=self.ParseCompoundStatement(body), - ) - delim = [ - t - for t in head - if t.ParenDepth == head.First().ParenDepth - and t.BraceDepth == head.First().BraceDepth - and t.Value == ":" - ] - if len(delim) != 1: - raise ActorCompilerError( - head.First().SourceLine, - "for statement must be 3-arg style or c++11 2-arg style", - ) - return RangeForStatement( - rangeExpression=self.str( - self.NormalizeWhitespace( - self.range(delim[0].Position + 1, head.End).SkipWhile( - lambda t: t.IsWhitespace - ) - ) - ), - rangeDecl=self.str( - self.NormalizeWhitespace( - self.range(head.Begin, delim[0].Position - 1).SkipWhile( - lambda t: t.IsWhitespace - ) - ) - ), - body=self.ParseCompoundStatement(self.range(head.End + 1, toks.End)), - ) - - def ParseIfStatement(self, toks: TokenRange) -> IfStatement: - toks = toks.Consume("if").SkipWhile(lambda t: t.IsWhitespace) - constexpr = toks.First().Value == "constexpr" - if constexpr: - toks = toks.Consume("constexpr").SkipWhile(lambda t: t.IsWhitespace) - expr = ( - toks.First(lambda t: not t.IsWhitespace) - .Assert("Expected (", lambda t: t.Value == "(") - .GetMatchingRangeIn(toks) - ) - return IfStatement( - expression=self.str(self.NormalizeWhitespace(expr)), - constexpr=constexpr, - ifBody=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), - ) - - def ParseElseStatement(self, toks: TokenRange, prev: Statement) -> None: - if_stmt = prev - while isinstance(if_stmt, IfStatement) and if_stmt.elseBody is not None: - if_stmt = if_stmt.elseBody - if not isinstance(if_stmt, IfStatement): - raise ActorCompilerError( - toks.First().SourceLine, "else without matching if" - ) - if_stmt.elseBody = self.ParseCompoundStatement(toks.Consume("else")) - - def ParseTryStatement(self, toks: TokenRange) -> TryStatement: - return TryStatement( - tryBody=self.ParseCompoundStatement(toks.Consume("try")), - catches=[], - ) - - def ParseCatchStatement(self, toks: TokenRange, prev: Statement) -> None: - if not isinstance(prev, TryStatement): - raise ActorCompilerError( - toks.First().SourceLine, "catch without matching try" - ) - expr = ( - toks.Consume("catch") - .First(lambda t: not t.IsWhitespace) - .Assert("Expected (", lambda t: t.Value == "(") - .GetMatchingRangeIn(toks) - ) - prev.catches.append( - TryStatement.Catch( - expression=self.str(self.NormalizeWhitespace(expr)), - body=self.ParseCompoundStatement(self.range(expr.End + 1, toks.End)), - FirstSourceLine=expr.First().SourceLine, - ) - ) - - IllegalKeywords = {"goto", "do", "finally", "__if_exists", "__if_not_exists"} - - def ParseCompoundStatement(self, toks: TokenRange) -> Statement: - nonWhitespace = lambda t: not t.IsWhitespace - first = toks.First(nonWhitespace) - if first.Value == "{": - inBraces = first.GetMatchingRangeIn(toks) - if ( - not self.range(inBraces.End, toks.End) - .Consume("}") - .All(lambda t: t.IsWhitespace) - ): - raise ActorCompilerError( - inBraces.Last().SourceLine, - "Unexpected tokens after compound statement", - ) - return self.ParseCodeBlock(inBraces) - statements: List[Statement] = [] - self.ParseStatement(toks.Skip(1), statements) - return statements[0] - - def ParseCodeBlock(self, toks: TokenRange) -> CodeBlock: - statements: List[Statement] = [] - while True: - delim = next( - ( - t - for t in toks - if t.ParenDepth == toks.First().ParenDepth - and t.BraceDepth == toks.First().BraceDepth - and t.Value in (";", "}") - ), - None, - ) - if delim is None: - break - self.ParseStatement(self.range(toks.Begin, delim.Position + 1), statements) - toks = self.range(delim.Position + 1, toks.End) - if not toks.All(lambda t: t.IsWhitespace): - raise ActorCompilerError( - toks.First(lambda t: not t.IsWhitespace).SourceLine, - "Trailing unterminated statement in code block", - ) - return CodeBlock(statements=statements) - - def ParseStatement(self, toks: TokenRange, statements: List[Statement]) -> None: - nonWhitespace = lambda t: not t.IsWhitespace - toks = toks.SkipWhile(lambda t: t.IsWhitespace) - - def Add(stmt: Statement) -> None: - stmt.FirstSourceLine = toks.First().SourceLine - statements.append(stmt) - - first_val = toks.First().Value - if first_val == "loop": - Add(self.ParseLoopStatement(toks)) - elif first_val == "while": - Add(self.ParseWhileStatement(toks)) - elif first_val == "for": - Add(self.ParseForStatement(toks)) - elif first_val == "break": - Add(BreakStatement()) - elif first_val == "continue": - Add(ContinueStatement()) - elif first_val == "return": - Add(self.ParseReturnStatement(toks)) - elif first_val == "{": - Add(self.ParseCompoundStatement(toks)) - elif first_val == "if": - Add(self.ParseIfStatement(toks)) - elif first_val == "else": - self.ParseElseStatement(toks, statements[-1]) - elif first_val == "choose": - Add(self.ParseChooseStatement(toks)) - elif first_val == "when": - Add(self.ParseWhenStatement(toks)) - elif first_val == "try": - Add(self.ParseTryStatement(toks)) - elif first_val == "catch": - self.ParseCatchStatement(toks, statements[-1]) - elif first_val == "throw": - Add(self.ParseThrowStatement(toks)) - else: - if first_val in self.IllegalKeywords: - raise ActorCompilerError( - toks.First().SourceLine, - f"Statement '{first_val}' not supported in actors.", - ) - if any(t.Value in ("wait", "waitNext") for t in toks): - Add(self.ParseWaitStatement(toks)) - elif first_val == "state": - Add(self.ParseStateDeclaration(toks)) - elif first_val == "switch" and any(t.Value == "return" for t in toks): - raise ActorCompilerError( - toks.First().SourceLine, - "Unsupported compound statement containing return.", - ) - elif first_val.startswith("#"): - raise ActorCompilerError( - toks.First().SourceLine, - f'Found "{first_val}". Preprocessor directives are not supported within ACTORs', - ) - else: - cleaned = toks.RevSkipWhile(lambda t: t.Value == ";") - if any(nonWhitespace(t) for t in cleaned): - Add( - PlainOldCodeStatement( - code=self.str(self.NormalizeWhitespace(cleaned)) + ";" - ) - ) - - def ParseClassContext(self, toks: TokenRange) -> Tuple[bool, str]: - name = "" - if toks.Begin == toks.End: - return False, name - - nonWhitespace = lambda t: not t.IsWhitespace - while True: - first = toks.First(nonWhitespace) - if first.Value == "[": - contents = first.GetMatchingRangeIn(toks) - toks = self.range(contents.End + 1, toks.End) - elif first.Value == "alignas": - toks = self.range(first.Position + 1, toks.End) - first = toks.First(nonWhitespace) - first.Assert("Expected ( after alignas", lambda t: t.Value == "(") - contents = first.GetMatchingRangeIn(toks) - toks = self.range(contents.End + 1, toks.End) - else: - break - - first = toks.First(nonWhitespace) - if not self.identifierPattern.match(first.Value): - return False, name - - while True: - first.Assert( - "Expected identifier", lambda t: self.identifierPattern.match(t.Value) - ) - name += first.Value - toks = self.range(first.Position + 1, toks.End) - next_token = toks.First(nonWhitespace) - if next_token.Value == "::": - name += "::" - toks = toks.SkipWhile(lambda t: t.IsWhitespace).Skip(1) - else: - break - first = toks.First(nonWhitespace) - - toks = toks.SkipWhile( - lambda t: t.IsWhitespace or t.Value in ("final", "explicit") - ) - first = toks.First(nonWhitespace) - if first.Value in (":", "{"): - return True, name - return False, "" - - def str(self, tokens: Iterable[Token]) -> str: - return "".join(tok.Value for tok in tokens) diff --git a/flow/actorcompiler/actorcompiler.csproj b/flow/actorcompiler/actorcompiler.csproj new file mode 100644 index 00000000000..ae866447097 --- /dev/null +++ b/flow/actorcompiler/actorcompiler.csproj @@ -0,0 +1,10 @@ + + + + Exe + net9.0 + false + true + + + diff --git a/flow/actorcompiler/parse_tree.py b/flow/actorcompiler/parse_tree.py deleted file mode 100644 index f1792eba07f..00000000000 --- a/flow/actorcompiler/parse_tree.py +++ /dev/null @@ -1,219 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import List, Optional, Sequence - - -@dataclass -class VarDeclaration: - type: str = "" - name: str = "" - initializer: Optional[str] = "" - initializerConstructorSyntax: bool = False - - -@dataclass -class Statement: - FirstSourceLine: int = 0 - - def containsWait(self) -> bool: - return False - - -@dataclass -class PlainOldCodeStatement(Statement): - code: str = "" - - def __str__(self) -> str: - return self.code - - -@dataclass -class StateDeclarationStatement(Statement): - decl: VarDeclaration = field(default_factory=VarDeclaration) - - def __str__(self) -> str: - if self.decl.initializerConstructorSyntax: - return f"State {self.decl.type} {self.decl.name}({self.decl.initializer});" - return f"State {self.decl.type} {self.decl.name} = {self.decl.initializer};" - - -@dataclass -class WhileStatement(Statement): - expression: str = "" - body: Statement = field(default_factory=Statement) - - def containsWait(self) -> bool: - return self.body.containsWait() - - -@dataclass -class ForStatement(Statement): - initExpression: str = "" - condExpression: str = "" - nextExpression: str = "" - body: Statement = field(default_factory=Statement) - - def containsWait(self) -> bool: - return self.body.containsWait() - - -@dataclass -class RangeForStatement(Statement): - rangeExpression: str = "" - rangeDecl: str = "" - body: Statement = field(default_factory=Statement) - - def containsWait(self) -> bool: - return self.body.containsWait() - - -@dataclass -class LoopStatement(Statement): - body: Statement = field(default_factory=Statement) - - def __str__(self) -> str: - return f"Loop {self.body}" - - def containsWait(self) -> bool: - return self.body.containsWait() - - -@dataclass -class BreakStatement(Statement): - pass - - -@dataclass -class ContinueStatement(Statement): - pass - - -@dataclass -class IfStatement(Statement): - expression: str = "" - constexpr: bool = False - ifBody: Statement = field(default_factory=Statement) - elseBody: Optional[Statement] = None - - def containsWait(self) -> bool: - return self.ifBody.containsWait() or ( - self.elseBody is not None and self.elseBody.containsWait() - ) - - -@dataclass -class ReturnStatement(Statement): - expression: str = "" - - def __str__(self) -> str: - return f"Return {self.expression}" - - -@dataclass -class WaitStatement(Statement): - result: VarDeclaration = field(default_factory=VarDeclaration) - futureExpression: str = "" - resultIsState: bool = False - isWaitNext: bool = False - - def __str__(self) -> str: - return f"Wait {self.result.type} {self.result.name} <- {self.futureExpression} ({'state' if self.resultIsState else 'local'})" - - def containsWait(self) -> bool: - return True - - -@dataclass -class ChooseStatement(Statement): - body: Statement = field(default_factory=Statement) - - def __str__(self) -> str: - return f"Choose {self.body}" - - def containsWait(self) -> bool: - return self.body.containsWait() - - -@dataclass -class WhenStatement(Statement): - wait: WaitStatement = field(default_factory=WaitStatement) - body: Optional[Statement] = None - - def __str__(self) -> str: - return f"When ({self.wait}) {self.body}" - - def containsWait(self) -> bool: - return True - - -@dataclass -class TryStatement(Statement): - @dataclass - class Catch: - expression: str = "" - body: Statement = field(default_factory=Statement) - FirstSourceLine: int = 0 - - tryBody: Statement = field(default_factory=Statement) - catches: List["TryStatement.Catch"] = field(default_factory=list) - - def containsWait(self) -> bool: - if self.tryBody.containsWait(): - return True - return any(c.body.containsWait() for c in self.catches) - - -@dataclass -class ThrowStatement(Statement): - expression: str = "" - - -@dataclass -class CodeBlock(Statement): - statements: Sequence[Statement] = field(default_factory=list) - - def __str__(self) -> str: - joined = "\n".join(str(stmt) for stmt in self.statements) - return f"CodeBlock\n{joined}\nEndCodeBlock" - - def containsWait(self) -> bool: - return any(stmt.containsWait() for stmt in self.statements) - - -@dataclass -class Declaration: - type: str = "" - name: str = "" - comment: str = "" - - -@dataclass -class Actor: - attributes: List[str] = field(default_factory=list) - returnType: Optional[str] = None - name: str = "" - enclosingClass: Optional[str] = None - parameters: Sequence[VarDeclaration] = field(default_factory=list) - templateFormals: Optional[Sequence[VarDeclaration]] = None - body: CodeBlock = field(default_factory=CodeBlock) - SourceLine: int = 0 - isStatic: bool = False - _isUncancellable: bool = False - testCaseParameters: Optional[str] = None - nameSpace: Optional[str] = None - isForwardDeclaration: bool = False - isTestCase: bool = False - - def IsCancellable(self) -> bool: - return self.returnType is not None and not self._isUncancellable - - def SetUncancellable(self) -> None: - self._isUncancellable = True - - -@dataclass -class Descr: - name: str = "" - superClassList: Optional[str] = None - body: List[Declaration] = field(default_factory=list) diff --git a/flow/actorcompiler/usertype.dat b/flow/actorcompiler/usertype.dat new file mode 100644 index 00000000000..77a51990089 --- /dev/null +++ b/flow/actorcompiler/usertype.dat @@ -0,0 +1,8 @@ +ACTOR +loop +state +choose +when +wait +waitNext +DESCR \ No newline at end of file diff --git a/flow/actorcompiler_py/Actor checklist.txt b/flow/actorcompiler_py/Actor checklist.txt new file mode 100644 index 00000000000..82ed087cefc --- /dev/null +++ b/flow/actorcompiler_py/Actor checklist.txt @@ -0,0 +1,84 @@ +Compile issues: + +- wait() must always assign the resulting value to a newly declared variable. + +- Variables used across a "wait() boundary" must be declared state + + +Remember to: + +- Add useful ASSERT()s + +- Add BUGGIFY() statements to expose rare cases to simulation + +- Add TEST() statements to any conditional/rare cases + +- Comment invariants, strategy, preconditions, tricky stuff when you + figure it out, even if it's not "your" code. + +- Factor common asynchronous control flows to use composition + of generic actors such as: + waitForAll + timeout + splitFuture + recurring + smartQuorum + AsyncMap + broadcast + &&, || + etc... + +- Declare classes NonCopyable unless they are, and you know what + that means + + +Run time issues: + +- Is the actor return type "void"? Make sure that some exception or timeout + will trigger eventually to clean up the actor. + +- If you send a future to another future, a long-lived forwardPromise + actor is created--make sure that the event happens eventually to free + this actor. + +- If you return a Future instead of a T from an actor, a forwardPromise + actor is created with the same lifetime issues as above. + +- Remember that parameters are internally passed to the actor as const & + and then copied into actor state variables + +- When you use *GetReply() or LoadBalance(), the "server" responding to + your request might get your request multiple times. + +- When you use getReply() instead of tryGetReply() you must ensure that the + actor will be cancelled if the service you are trying to connect to is + no longer available. (Otherwise, an infinite waiting loop) + +- For each wait: + + - An actor_cancelled exception can be thrown if the actor's return + value future is dropped. + + - An exception can arrive instead of a value + + - What happens if it never returns? + + - If the client fulfilling the wait is coming over the network, you + might get the same request multiple times + + +Performance issues: + +- Wait a little extra time before doing something time-consuming or + irreversible to see if it is still necessary. + +- When waiting for a number of things, wait a little extra time to get + the stragglers. (See the SmartQuorum() generic actor) + +- If asking another asynchronous server to do units of work, don't queue up more + work than is necessary to keep the server busy. Likewise, if you are + busy, let your own work queue fill up to signal your requester + that you are blocked. Also do this personally with managers assigning + you stuff. + +- Pass all variables as "const &" if their size is greater than 8 bytes. diff --git a/flow/actorcompiler/main.py b/flow/actorcompiler_py/__main__.py similarity index 84% rename from flow/actorcompiler/main.py rename to flow/actorcompiler_py/__main__.py index 37a3f5a2d92..b6a03591a27 100644 --- a/flow/actorcompiler/main.py +++ b/flow/actorcompiler_py/__main__.py @@ -1,3 +1,7 @@ +""" +Allow running the actorcompiler as a module: + python3 -m flow.actorcompiler input.actor.cpp output.g.cpp +""" from __future__ import annotations import argparse @@ -6,8 +10,8 @@ import sys from pathlib import Path -from . import ActorCompilerError from .actor_parser import ActorParser, ErrorMessagePolicy +from .errors import ActorCompilerError def overwrite_by_move(target: Path, temporary: Path) -> None: @@ -18,7 +22,7 @@ def overwrite_by_move(target: Path, temporary: Path) -> None: target.chmod(stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) -def parse_arguments(argv: list[str]) -> argparse.Namespace: +def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser( prog="actorcompiler", description="Python port of the Flow actor compiler", @@ -30,24 +34,22 @@ def parse_arguments(argv: list[str]) -> argparse.Namespace: parser.add_argument("--disable-diagnostics", action="store_true") parser.add_argument("--generate-probes", action="store_true") parser.add_argument("--help", action="help", help=argparse.SUPPRESS) - args = parser.parse_args(argv) + args = parser.parse_args() if not args.input or not args.output: parser.print_usage(sys.stderr) sys.exit(100) return args -def main(argv: list[str] | None = None) -> int: - if argv is None: - argv = sys.argv[1:] - args = parse_arguments(argv) +def main() -> int: + args = parse_arguments() input_path = Path(args.input) output_path = Path(args.output) output_tmp = output_path.with_suffix(output_path.suffix + ".tmp") output_uid = output_path.with_suffix(output_path.suffix + ".uid") policy = ErrorMessagePolicy() - policy.DisableDiagnostics = args.disable_diagnostics + policy.disable_diagnostics = args.disable_diagnostics try: print("actorcompiler", " ".join(sys.argv[1:])) @@ -60,11 +62,11 @@ def main(argv: list[str] | None = None) -> int: ) with output_tmp.open("w", newline="\n") as out_file: - parser.Write(out_file, str(output_path).replace("\\", "/")) + parser.write(out_file, str(output_path).replace("\\", "/")) overwrite_by_move(output_path, output_tmp) with output_tmp.open("w", newline="\n") as uid_file: - for (hi, lo), value in parser.uidObjects.items(): + for (hi, lo), value in parser.uid_objects.items(): uid_file.write(f"{hi}|{lo}|{value}\n") overwrite_by_move(output_uid, output_tmp) @@ -81,6 +83,8 @@ def main(argv: list[str] | None = None) -> int: output_path.unlink() return 1 except Exception as exc: # pylint: disable=broad-except + import traceback + traceback.print_exc() print( f"{input_path}(1): error FAC2000: Internal {exc}", file=sys.stderr, diff --git a/flow/actorcompiler/actor_compiler.py b/flow/actorcompiler_py/actor_compiler.py similarity index 63% rename from flow/actorcompiler/actor_compiler.py rename to flow/actorcompiler_py/actor_compiler.py index 7ac4c5603c2..5a6a6c6e9a4 100644 --- a/flow/actorcompiler/actor_compiler.py +++ b/flow/actorcompiler_py/actor_compiler.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import Dict, Iterable, List, Optional, Sequence, Tuple -from . import ActorCompilerError +from .errors import ActorCompilerError from .parse_tree import ( Actor, BreakStatement, @@ -39,15 +39,15 @@ def __init__( formal_parameters: Optional[Sequence[str]] = None, ) -> None: self.name = name - self.returnType = return_type - self.formalParameters = list(formal_parameters or []) - self.endIsUnreachable = False - self.exceptionParameterIs: Optional[str] = None - self.publicName = False + self.return_type = return_type + self.formal_parameters = list(formal_parameters or []) + self.end_is_unreachable = False + self.exception_parameter_is: Optional[str] = None + self.public_name = False self.specifiers = "" self._indentation = "" self._body = io.StringIO() - self.wasCalled = False + self.was_called = False self.overload: Optional["Function"] = None def set_overload(self, overload: "Function") -> None: @@ -61,10 +61,10 @@ def pop_overload(self) -> Optional["Function"]: def add_overload(self, *formal_parameters: str) -> None: overload = Function( name=self.name, - return_type=self.returnType, + return_type=self.return_type, formal_parameters=formal_parameters, ) - overload.endIsUnreachable = self.endIsUnreachable + overload.end_is_unreachable = self.end_is_unreachable overload._indentation = self._indentation self.set_overload(overload) @@ -96,8 +96,8 @@ def body_text(self) -> str: return value def use_by_name(self) -> str: - self.wasCalled = True - return self.name if self.publicName else f"a_{self.name}" + self.was_called = True + return self.name if self.public_name else f"a_{self.name}" def call(self, *parameters: str) -> str: params = ", ".join(parameters) @@ -111,7 +111,7 @@ def __init__(self) -> None: def call(self, *parameters: str) -> str: if parameters: raise ActorCompilerError(0, "LiteralBreak called with parameters!") - self.wasCalled = True + self.was_called = True return "break" @@ -122,7 +122,7 @@ def __init__(self) -> None: def call(self, *parameters: str) -> str: if parameters: raise ActorCompilerError(0, "LiteralContinue called with parameters!") - self.wasCalled = True + self.was_called = True return "continue" @@ -131,15 +131,15 @@ class StateVar: type: str = "" name: str = "" initializer: Optional[str] = None - initializerConstructorSyntax: bool = False - SourceLine: int = 0 + initializer_constructor_syntax: bool = False + source_line: int = 0 @dataclass class CallbackVar: type: str = "" - CallbackGroup: int = 0 - SourceLine: int = 0 + callback_group: int = 0 + source_line: int = 0 class Context: @@ -147,17 +147,17 @@ def __init__( self, target: Optional[Function] = None, next_func: Optional[Function] = None, - breakF: Optional[Function] = None, - continueF: Optional[Function] = None, - catchFErr: Optional[Function] = None, - tryLoopDepth: int = 0, + break_f: Optional[Function] = None, + continue_f: Optional[Function] = None, + catch_f_err: Optional[Function] = None, + try_loop_depth: int = 0, ) -> None: self.target = target self.next = next_func - self.breakF = breakF - self.continueF = continueF - self.catchFErr = catchFErr - self.tryLoopDepth = tryLoopDepth + self.break_f = break_f + self.continue_f = continue_f + self.catch_f_err = catch_f_err + self.try_loop_depth = try_loop_depth def unreachable(self) -> None: self.target = None @@ -165,52 +165,52 @@ def unreachable(self) -> None: def with_target(self, new_target: Function) -> "Context": return Context( target=new_target, - breakF=self.breakF, - continueF=self.continueF, - catchFErr=self.catchFErr, - tryLoopDepth=self.tryLoopDepth, + break_f=self.break_f, + continue_f=self.continue_f, + catch_f_err=self.catch_f_err, + try_loop_depth=self.try_loop_depth, ) def loop_context( self, new_target: Function, - breakF: Function, - continueF: Function, + break_f: Function, + continue_f: Function, deltaLoopDepth: int, ) -> "Context": return Context( target=new_target, - breakF=breakF, - continueF=continueF, - catchFErr=self.catchFErr, - tryLoopDepth=self.tryLoopDepth + deltaLoopDepth, + break_f=break_f, + continue_f=continue_f, + catch_f_err=self.catch_f_err, + try_loop_depth=self.try_loop_depth + deltaLoopDepth, ) def with_catch(self, new_catch: Function) -> "Context": return Context( target=self.target, - breakF=self.breakF, - continueF=self.continueF, - catchFErr=new_catch, + break_f=self.break_f, + continue_f=self.continue_f, + catch_f_err=new_catch, ) def clone(self) -> "Context": return Context( target=self.target, next_func=self.next, - breakF=self.breakF, - continueF=self.continueF, - catchFErr=self.catchFErr, - tryLoopDepth=self.tryLoopDepth, + break_f=self.break_f, + continue_f=self.continue_f, + catch_f_err=self.catch_f_err, + try_loop_depth=self.try_loop_depth, ) def copy_from(self, other: "Context") -> None: self.target = other.target self.next = other.next - self.breakF = other.breakF - self.continueF = other.continueF - self.catchFErr = other.catchFErr - self.tryLoopDepth = other.tryLoopDepth + self.break_f = other.break_f + self.continue_f = other.continue_f + self.catch_f_err = other.catch_f_err + self.try_loop_depth = other.try_loop_depth class DescrCompiler: @@ -258,9 +258,9 @@ def write(self, writer) -> int: f"{indent}\ttypedef make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type field_indexes;\n" ) writer.write(f"{indent}}};\n") - if self.descr.superClassList: + if self.descr.super_class_list: writer.write( - f"{indent}struct {self.descr.name} : {self.descr.superClassList} {{\n" + f"{indent}struct {self.descr.name} : {self.descr.super_class_list} {{\n" ) else: writer.write(f"{indent}struct {self.descr.name} {{\n") @@ -280,10 +280,10 @@ def write(self, writer) -> int: class ActorCompiler: member_indent_str = "\t" - loopDepth0 = "int loopDepth=0" - loopDepth = "int loopDepth" - codeIndent = 2 - usedClassNames: set[str] = set() + loop_depth_0 = "int loopDepth=0" + loop_depth = "int loopDepth" + code_indent = 2 + used_class_names: set[str] = set() def __init__( self, @@ -294,96 +294,96 @@ def __init__( generate_probes: bool, ) -> None: self.actor = actor - self.sourceFile = source_file - self.isTopLevel = is_top_level - self.LineNumbersEnabled = line_numbers_enabled - self.generateProbes = generate_probes - self.className = "" - self.fullClassName = "" - self.stateClassName = "" + self.source_file = source_file + self.is_top_level = is_top_level + self.line_numbers_enabled = line_numbers_enabled + self.generate_probes = generate_probes + self.class_name = "" + self.full_class_name = "" + self.state_class_name = "" self.state: List[StateVar] = [] self.callbacks: List[CallbackVar] = [] - self.chooseGroups = 0 - self.whenCount = 0 - self.This = "" - self.uidObjects: Dict[Tuple[int, int], str] = {} + self.choose_groups = 0 + self.when_count = 0 + self.this = "" + self.uid_objects: Dict[Tuple[int, int], str] = {} self.functions: Dict[str, Function] = {} self.iterators: Dict[str, int] = {} - self.FindState() + self.find_state() - def ByteToLong(self, data: bytes) -> int: + def byte_to_long(self, data: bytes) -> int: result = 0 for b in data: result += b result <<= 8 return result & ((1 << 64) - 1) - def GetUidFromString(self, value: str) -> Tuple[int, int]: + def get_uid_from_string(self, value: str) -> Tuple[int, int]: digest = hashlib.sha256(value.encode("utf-8")).digest() - first = self.ByteToLong(digest[:8]) - second = self.ByteToLong(digest[8:16]) + first = self.byte_to_long(digest[:8]) + second = self.byte_to_long(digest[8:16]) return (first, second) - def WriteActorFunction(self, writer, full_return_type: str) -> None: - self.WriteTemplate(writer) - self.LineNumber(writer, self.actor.SourceLine) + def writeActorFunction(self, writer, full_return_type: str) -> None: + self.writeTemplate(writer) + self.line_number(writer, self.actor.source_line) for attribute in self.actor.attributes: writer.write(f"{attribute} ") - if self.actor.isStatic: + if self.actor.is_static: writer.write("static ") namespace_prefix = ( - "" if self.actor.nameSpace is None else f"{self.actor.nameSpace}::" + "" if self.actor.name_space is None else f"{self.actor.name_space}::" ) - params = ", ".join(self.ParameterList()) + params = ", ".join(self.parameter_list()) writer.write( f"{full_return_type} {namespace_prefix}{self.actor.name}( {params} ) {{\n" ) - self.LineNumber(writer, self.actor.SourceLine) + self.line_number(writer, self.actor.source_line) ctor_args = ", ".join(param.name for param in self.actor.parameters) - new_actor = f"new {self.fullClassName}({ctor_args})" - if self.actor.returnType is not None: - writer.write(f"\treturn Future<{self.actor.returnType}>({new_actor});\n") + new_actor = f"new {self.full_class_name}({ctor_args})" + if self.actor.return_type is not None: + writer.write(f"\treturn Future<{self.actor.return_type}>({new_actor});\n") else: writer.write(f"\t{new_actor};\n") writer.write("}\n") - def WriteActorClass( + def writeActorClass( self, writer, full_state_class_name: str, body: Function ) -> None: writer.write( f"// This generated class is to be used only via {self.actor.name}()\n" ) - self.WriteTemplate(writer) - self.LineNumber(writer, self.actor.SourceLine) + self.writeTemplate(writer) + self.line_number(writer, self.actor.source_line) callback_bases = ", ".join(f"public {cb.type}" for cb in self.callbacks) if callback_bases: callback_bases += ", " writer.write( "class {0} final : public Actor<{2}>, {3}public FastAllocated<{1}>, public {4} {{\n".format( - self.className, - self.fullClassName, - "void" if self.actor.returnType is None else self.actor.returnType, + self.class_name, + self.full_class_name, + "void" if self.actor.return_type is None else self.actor.return_type, callback_bases, full_state_class_name, ) ) writer.write("public:\n") - writer.write(f"\tusing FastAllocated<{self.fullClassName}>::operator new;\n") - writer.write(f"\tusing FastAllocated<{self.fullClassName}>::operator delete;\n") - actor_identifier_key = f"{self.sourceFile}:{self.actor.name}" - uid = self.GetUidFromString(actor_identifier_key) - self.uidObjects[(uid[0], uid[1])] = actor_identifier_key + writer.write(f"\tusing FastAllocated<{self.full_class_name}>::operator new;\n") + writer.write(f"\tusing FastAllocated<{self.full_class_name}>::operator delete;\n") + actor_identifier_key = f"{self.source_file}:{self.actor.name}" + uid = self.get_uid_from_string(actor_identifier_key) + self.uid_objects[(uid[0], uid[1])] = actor_identifier_key writer.write( f"\tstatic constexpr ActorIdentifier __actorIdentifier = UID({uid[0]}UL, {uid[1]}UL);\n" ) writer.write("\tActiveActorHelper activeActorHelper;\n") writer.write("#pragma clang diagnostic push\n") writer.write('#pragma clang diagnostic ignored "-Wdelete-non-virtual-dtor"\n') - if self.actor.returnType is not None: + if self.actor.return_type is not None: writer.write(" void destroy() override {\n") writer.write(" activeActorHelper.~ActiveActorHelper();\n") writer.write( - f" static_cast*>(this)->~Actor();\n" + f" static_cast*>(this)->~Actor();\n" ) writer.write(" operator delete(this);\n") writer.write(" }\n") @@ -396,15 +396,15 @@ def WriteActorClass( writer.write("#pragma clang diagnostic pop\n") for cb in self.callbacks: writer.write(f"friend struct {cb.type};\n") - self.LineNumber(writer, self.actor.SourceLine) - self.WriteConstructor(body, writer, full_state_class_name) - self.WriteCancelFunc(writer) + self.line_number(writer, self.actor.source_line) + self.writeConstructor(body, writer, full_state_class_name) + self.writeCancelFunc(writer) writer.write("};\n") - def Write(self, writer) -> None: + def write(self, writer) -> None: full_return_type = ( - f"Future<{self.actor.returnType}>" - if self.actor.returnType is not None + f"Future<{self.actor.return_type}>" + if self.actor.return_type is not None else "void" ) for i in range(1 << 16): @@ -413,109 +413,109 @@ def Write(self, writer) -> None: self.actor.name[1:], str(i) if i != 0 else "", ( - self.actor.enclosingClass.replace("::", "_") + "_" - if self.actor.enclosingClass is not None - and self.actor.isForwardDeclaration + self.actor.enclosing_class.replace("::", "_") + "_" + if self.actor.enclosing_class is not None + and self.actor.is_forward_declaration else ( - self.actor.nameSpace.replace("::", "_") + "_" - if self.actor.nameSpace is not None + self.actor.name_space.replace("::", "_") + "_" + if self.actor.name_space is not None else "" ) ), ) - if self.actor.isForwardDeclaration: - self.className = class_name + if self.actor.is_forward_declaration: + self.class_name = class_name break - if class_name not in ActorCompiler.usedClassNames: - self.className = class_name - ActorCompiler.usedClassNames.add(class_name) + if class_name not in ActorCompiler.used_class_names: + self.class_name = class_name + ActorCompiler.used_class_names.add(class_name) break - self.fullClassName = self.className + self.GetTemplateActuals() - actor_class_formal = VarDeclaration(name=self.className, type="class") - self.This = f"static_cast<{actor_class_formal.name}*>(this)" - self.stateClassName = self.className + "State" - full_state = self.stateClassName + self.GetTemplateActuals( - VarDeclaration(type="class", name=self.fullClassName) + self.full_class_name = self.class_name + self.get_template_actuals() + actor_class_formal = VarDeclaration(name=self.class_name, type="class") + self.this = f"static_cast<{actor_class_formal.name}*>(this)" + self.state_class_name = self.class_name + "State" + full_state = self.state_class_name + self.get_template_actuals( + VarDeclaration(type="class", name=self.full_class_name) ) - if self.actor.isForwardDeclaration: + if self.actor.is_forward_declaration: for attribute in self.actor.attributes: writer.write(f"{attribute} ") - if self.actor.isStatic: + if self.actor.is_static: writer.write("static ") - ns = "" if self.actor.nameSpace is None else f"{self.actor.nameSpace}::" + ns = "" if self.actor.name_space is None else f"{self.actor.name_space}::" writer.write( - f"{full_return_type} {ns}{self.actor.name}( {', '.join(self.ParameterList())} );\n" + f"{full_return_type} {ns}{self.actor.name}( {', '.join(self.parameter_list())} );\n" ) - if self.actor.enclosingClass is not None: - writer.write(f"template friend class {self.stateClassName};\n") + if self.actor.enclosing_class is not None: + writer.write(f"template friend class {self.state_class_name};\n") return - body = self.getFunction("", "body", self.loopDepth0) + body = self.get_function("", "body", self.loop_depth_0) body_context = Context( target=body, - catchFErr=self.getFunction( - body.name, "Catch", "Error error", self.loopDepth0 + catch_f_err=self.get_function( + body.name, "Catch", "Error error", self.loop_depth_0 ), ) - end_context = self.TryCatchCompile(self.actor.body, body_context) + end_context = self.try_catch_compile(self.actor.body, body_context) if end_context.target is not None: - if self.actor.returnType is None: + if self.actor.return_type is None: self.compile_statement( ReturnStatement( - FirstSourceLine=self.actor.SourceLine, expression="" + first_source_line=self.actor.source_line, expression="" ), end_context, ) else: raise ActorCompilerError( - self.actor.SourceLine, + self.actor.source_line, "Actor {0} fails to return a value", self.actor.name, ) - if self.actor.returnType is not None: - body_context.catchFErr.write_line(f"this->~{self.stateClassName}();") - body_context.catchFErr.write_line( - f"{self.This}->sendErrorAndDelPromiseRef(error);" + if self.actor.return_type is not None: + body_context.catch_f_err.write_line(f"this->~{self.state_class_name}();") + body_context.catch_f_err.write_line( + f"{self.this}->sendErrorAndDelPromiseRef(error);" ) else: - body_context.catchFErr.write_line(f"delete {self.This};") - body_context.catchFErr.write_line("loopDepth = 0;") - if self.isTopLevel and self.actor.nameSpace is None: + body_context.catch_f_err.write_line(f"delete {self.this};") + body_context.catch_f_err.write_line("loopDepth = 0;") + if self.is_top_level and self.actor.name_space is None: writer.write("namespace {\n") writer.write( f"// This generated class is to be used only via {self.actor.name}()\n" ) - self.WriteTemplate(writer, actor_class_formal) - self.LineNumber(writer, self.actor.SourceLine) - writer.write(f"class {self.stateClassName} {{\n") + self.writeTemplate(writer, actor_class_formal) + self.line_number(writer, self.actor.source_line) + writer.write(f"class {self.state_class_name} {{\n") writer.write("public:\n") - self.LineNumber(writer, self.actor.SourceLine) - self.WriteStateConstructor(writer) - self.WriteStateDestructor(writer) - self.WriteFunctions(writer) + self.line_number(writer, self.actor.source_line) + self.writeStateConstructor(writer) + self.writeStateDestructor(writer) + self.writeFunctions(writer) for st in self.state: - self.LineNumber(writer, st.SourceLine) + self.line_number(writer, st.source_line) writer.write(f"\t{st.type} {st.name};\n") writer.write("};\n") - self.WriteActorClass(writer, full_state, body) - if self.isTopLevel and self.actor.nameSpace is None: + self.writeActorClass(writer, full_state, body) + if self.is_top_level and self.actor.name_space is None: writer.write("} // namespace\n") - self.WriteActorFunction(writer, full_return_type) - if self.actor.testCaseParameters is not None: + self.writeActorFunction(writer, full_return_type) + if self.actor.test_case_parameters is not None: writer.write( - f"ACTOR_TEST_CASE({self.actor.name}, {self.actor.testCaseParameters})\n" + f"ACTOR_TEST_CASE({self.actor.name}, {self.actor.test_case_parameters})\n" ) thisAddress = "reinterpret_cast(this)" - def ProbeEnter(self, fun: Function, name: str, index: int = -1) -> None: - if self.generateProbes: + def probe_enter(self, fun: Function, name: str, index: int = -1) -> None: + if self.generate_probes: fun.write_line( 'fdb_probe_actor_enter("{0}", {1}, {2});', name, self.thisAddress, index, ) - block_identifier = self.GetUidFromString(fun.name) + block_identifier = self.get_uid_from_string(fun.name) fun.write_line("#ifdef WITH_ACAC") fun.write_line( "static constexpr ActorBlockIdentifier __identifier = UID({0}UL, {1}UL);", @@ -524,12 +524,12 @@ def ProbeEnter(self, fun: Function, name: str, index: int = -1) -> None: ) fun.write_line( "ActorExecutionContextHelper __helper(static_cast<{0}*>(this)->activeActorHelper.actorID, __identifier);", - self.className, + self.class_name, ) fun.write_line("#endif // WITH_ACAC") - def ProbeExit(self, fun: Function, name: str, index: int = -1) -> None: - if self.generateProbes: + def probe_exit(self, fun: Function, name: str, index: int = -1) -> None: + if self.generate_probes: fun.write_line( 'fdb_probe_actor_exit("{0}", {1}, {2});', name, @@ -537,41 +537,41 @@ def ProbeExit(self, fun: Function, name: str, index: int = -1) -> None: index, ) - def ProbeCreate(self, fun: Function, name: str) -> None: - if self.generateProbes: + def probe_create(self, fun: Function, name: str) -> None: + if self.generate_probes: fun.write_line( 'fdb_probe_actor_create("{0}", {1});', name, self.thisAddress ) - def ProbeDestroy(self, fun: Function, name: str) -> None: - if self.generateProbes: + def probe_destroy(self, fun: Function, name: str) -> None: + if self.generate_probes: fun.write_line( 'fdb_probe_actor_destroy("{0}", {1});', name, self.thisAddress, ) - def LineNumber(self, writer, source_line: int) -> None: + def line_number(self, writer, source_line: int) -> None: if source_line == 0: raise ActorCompilerError(0, "Invalid source line (0)") - if self.LineNumbersEnabled: + if self.line_numbers_enabled: writer.write( '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( - source_line, self.sourceFile + source_line, self.source_file ) ) - def LineNumberFunction(self, func: Function, source_line: int) -> None: + def line_numberFunction(self, func: Function, source_line: int) -> None: if source_line == 0: raise ActorCompilerError(0, "Invalid source line (0)") - if self.LineNumbersEnabled: + if self.line_numbers_enabled: func.write_line_unindented( '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"'.format( - source_line, self.sourceFile + source_line, self.source_file ) ) - def TryCatch( + def try_catch( self, cx: Context, catch_function: Optional[Function], @@ -591,7 +591,7 @@ def TryCatch( cx.target.write_line( "\tloopDepth = {0};", catch_function.call( - "error", self.AdjustLoopDepth(catch_loop_depth) + "error", self.adjust_loop_depth(catch_loop_depth) ), ) else: @@ -601,7 +601,7 @@ def TryCatch( cx.target.write_line( "\tloopDepth = {0};", catch_function.call( - "unknown_error()", self.AdjustLoopDepth(catch_loop_depth) + "unknown_error()", self.adjust_loop_depth(catch_loop_depth) ), ) else: @@ -610,87 +610,87 @@ def TryCatch( ) cx.target.write_line("}") - def TryCatchCompile(self, block: CodeBlock, cx: Context) -> Context: + def try_catch_compile(self, block: CodeBlock, cx: Context) -> Context: result_holder = {"ctx": cx} def action() -> None: result_holder["ctx"] = self._try_catch_compile_body(block, cx) - self.TryCatch(cx, cx.catchFErr, cx.tryLoopDepth, action) + self.try_catch(cx, cx.catch_f_err, cx.try_loop_depth, action) return result_holder["ctx"] def _try_catch_compile_body(self, block: CodeBlock, cx: Context) -> Context: - compiled = self.Compile(block, cx, True) + compiled = self.compile(block, cx, True) if compiled.target is not None: - next_func = self.getFunction(compiled.target.name, "cont", self.loopDepth) + next_func = self.get_function(compiled.target.name, "cont", self.loop_depth) compiled.target.write_line("loopDepth = {0};", next_func.call("loopDepth")) compiled.target = next_func compiled.next = None return compiled - def WriteTemplate(self, writer, *extra_parameters: VarDeclaration) -> None: - formals = list(self.actor.templateFormals or []) + list(extra_parameters) + def writeTemplate(self, writer, *extra_parameters: VarDeclaration) -> None: + formals = list(self.actor.template_formals or []) + list(extra_parameters) if not formals: return - self.LineNumber(writer, self.actor.SourceLine) + self.line_number(writer, self.actor.source_line) writer.write( "template <{0}>\n".format(", ".join(f"{p.type} {p.name}" for p in formals)) ) - def GetTemplateActuals(self, *extra_parameters: VarDeclaration) -> str: - formals = list(self.actor.templateFormals or []) + list(extra_parameters) + def get_template_actuals(self, *extra_parameters: VarDeclaration) -> str: + formals = list(self.actor.template_formals or []) + list(extra_parameters) if not formals: return "" return "<{0}>".format(", ".join(p.name for p in formals)) - def WillContinue(self, stmt: Statement) -> bool: + def will_continue(self, stmt: Statement) -> bool: return any( isinstance(sub, (ChooseStatement, WaitStatement, TryStatement)) - for sub in self.Flatten(stmt) + for sub in self.flatten(stmt) ) - def AsCodeBlock(self, statement: Statement) -> CodeBlock: + def as_code_block(self, statement: Statement) -> CodeBlock: if isinstance(statement, CodeBlock): return statement return CodeBlock(statements=[statement]) - def Flatten(self, stmt: Optional[Statement]) -> Iterable[Statement]: + def flatten(self, stmt: Optional[Statement]) -> Iterable[Statement]: if stmt is None: return [] def _flatten(s: Statement) -> Iterable[Statement]: yield s if isinstance(s, LoopStatement): - yield from self.Flatten(s.body) + yield from self.flatten(s.body) elif isinstance(s, WhileStatement): - yield from self.Flatten(s.body) + yield from self.flatten(s.body) elif isinstance(s, ForStatement): - yield from self.Flatten(s.body) + yield from self.flatten(s.body) elif isinstance(s, RangeForStatement): - yield from self.Flatten(s.body) + yield from self.flatten(s.body) elif isinstance(s, CodeBlock): for child in s.statements: - yield from self.Flatten(child) + yield from self.flatten(child) elif isinstance(s, IfStatement): - yield from self.Flatten(s.ifBody) - if s.elseBody: - yield from self.Flatten(s.elseBody) + yield from self.flatten(s.if_body) + if s.else_body: + yield from self.flatten(s.else_body) elif isinstance(s, ChooseStatement): - yield from self.Flatten(s.body) + yield from self.flatten(s.body) elif isinstance(s, WhenStatement): if s.body: - yield from self.Flatten(s.body) + yield from self.flatten(s.body) elif isinstance(s, TryStatement): - yield from self.Flatten(s.tryBody) + yield from self.flatten(s.try_body) for c in s.catches: - yield from self.Flatten(c.body) + yield from self.flatten(c.body) return list(_flatten(stmt)) - def FindState(self) -> None: + def find_state(self) -> None: self.state = [ StateVar( - SourceLine=self.actor.SourceLine, + source_line=self.actor.source_line, name=p.name, type=p.type, initializer=p.name, @@ -698,12 +698,12 @@ def FindState(self) -> None: for p in self.actor.parameters ] - def AdjustLoopDepth(self, subtract: int) -> str: + def adjust_loop_depth(self, subtract: int) -> str: if subtract == 0: return "loopDepth" return f"std::max(0, loopDepth - {subtract})" - def ParameterList(self) -> List[str]: + def parameter_list(self) -> List[str]: params = [] for p in self.actor.parameters: if p.initializer: @@ -712,7 +712,7 @@ def ParameterList(self) -> List[str]: params.append(f"{p.type} const& {p.name}") return params - def getFunction( + def get_function( self, base_name: str, add_name: str, @@ -738,81 +738,81 @@ def getFunction( return_type="int", formal_parameters=params, ) - func.indent(self.codeIndent) + func.indent(self.code_indent) if overload_formal_parameters: func.add_overload(*overload_formal_parameters) self.functions[func.name] = func return func - def WriteFunctions(self, writer) -> None: + def writeFunctions(self, writer) -> None: for func in self.functions.values(): body = func.body_text if body: - self.WriteFunction(writer, func, body) + self.writeFunction(writer, func, body) if func.overload: overload_body = func.overload.body_text if overload_body: - self.WriteFunction(writer, func.overload, overload_body) + self.writeFunction(writer, func.overload, overload_body) - def WriteFunction(self, writer, func: Function, body: str) -> None: + def writeFunction(self, writer, func: Function, body: str) -> None: spec = "" if not func.specifiers else f" {func.specifiers}" trailing = " " if spec == "" else "" signature = ( f"{self.member_indent_str}" - f"{'' if func.returnType == '' else f'{func.returnType} '}" - f"{func.use_by_name()}({','.join(func.formalParameters)})" + f"{'' if func.return_type == '' else f'{func.return_type} '}" + f"{func.use_by_name()}({','.join(func.formal_parameters)})" f"{spec}{trailing}\n" ) writer.write(signature) - if func.returnType != "": + if func.return_type != "": writer.write(f"{self.member_indent_str}{{\n") writer.write(body) writer.write("\n") - if not func.endIsUnreachable: + if not func.end_is_unreachable: writer.write(f"{self.member_indent_str}\treturn loopDepth;\n") writer.write(f"{self.member_indent_str}}}\n") - def WriteCancelFunc(self, writer) -> None: - if not self.actor.IsCancellable(): + def writeCancelFunc(self, writer) -> None: + if not self.actor.is_cancellable(): return cancel_func = Function( name="cancel", return_type="void", formal_parameters=[], ) - cancel_func.endIsUnreachable = True - cancel_func.publicName = True + cancel_func.end_is_unreachable = True + cancel_func.public_name = True cancel_func.specifiers = "override" - cancel_func.indent(self.codeIndent) + cancel_func.indent(self.code_indent) cancel_func.write_line("auto wait_state = this->actor_wait_state;") cancel_func.write_line("this->actor_wait_state = -1;") cancel_func.write_line("switch (wait_state) {") last_group = -1 - for cb in sorted(self.callbacks, key=lambda c: c.CallbackGroup): - if cb.CallbackGroup != last_group: - last_group = cb.CallbackGroup + for cb in sorted(self.callbacks, key=lambda c: c.callback_group): + if cb.callback_group != last_group: + last_group = cb.callback_group cancel_func.write_line( "case {0}: this->a_callback_error(({1}*)0, actor_cancelled()); break;", - cb.CallbackGroup, + cb.callback_group, cb.type, ) cancel_func.write_line("}") - self.WriteFunction(writer, cancel_func, cancel_func.body_text) + self.writeFunction(writer, cancel_func, cancel_func.body_text) - def WriteConstructor( + def writeConstructor( self, body: Function, writer, full_state_class_name: str ) -> None: constructor = Function( - name=self.className, + name=self.class_name, return_type="", - formal_parameters=self.ParameterList(), + formal_parameters=self.parameter_list(), ) - constructor.endIsUnreachable = True - constructor.publicName = True - constructor.indent(self.codeIndent) + constructor.end_is_unreachable = True + constructor.public_name = True + constructor.indent(self.code_indent) constructor.write_line( " : Actor<{0}>(),".format( - "void" if self.actor.returnType is None else self.actor.returnType + "void" if self.actor.return_type is None else self.actor.return_type ) ) constructor.write_line( @@ -825,80 +825,80 @@ def WriteConstructor( constructor.indent(-1) constructor.write_line("{") constructor.indent(+1) - self.ProbeEnter(constructor, self.actor.name) + self.probe_enter(constructor, self.actor.name) constructor.write_line("#ifdef ENABLE_SAMPLING") constructor.write_line('this->lineage.setActorName("{0}");', self.actor.name) constructor.write_line("LineageScope _(&this->lineage);") constructor.write_line("#endif") constructor.write_line("this->{0};", body.call()) - self.ProbeExit(constructor, self.actor.name) - self.WriteFunction(writer, constructor, constructor.body_text) + self.probe_exit(constructor, self.actor.name) + self.writeFunction(writer, constructor, constructor.body_text) - def WriteStateConstructor(self, writer) -> None: + def writeStateConstructor(self, writer) -> None: constructor = Function( - name=self.stateClassName, + name=self.state_class_name, return_type="", - formal_parameters=self.ParameterList(), + formal_parameters=self.parameter_list(), ) - constructor.endIsUnreachable = True - constructor.publicName = True - constructor.indent(self.codeIndent) + constructor.end_is_unreachable = True + constructor.public_name = True + constructor.indent(self.code_indent) ini = None - line = self.actor.SourceLine + line = self.actor.source_line for state_var in self.state: if state_var.initializer is None: continue - self.LineNumberFunction(constructor, line) + self.line_numberFunction(constructor, line) if ini is None: ini = " : " else: constructor.write_line(ini + ",") ini = " " ini += f"{state_var.name}({state_var.initializer})" - line = state_var.SourceLine - self.LineNumberFunction(constructor, line) + line = state_var.source_line + self.line_numberFunction(constructor, line) if ini: constructor.write_line(ini) constructor.indent(-1) constructor.write_line("{") constructor.indent(+1) - self.ProbeCreate(constructor, self.actor.name) - self.WriteFunction(writer, constructor, constructor.body_text) + self.probe_create(constructor, self.actor.name) + self.writeFunction(writer, constructor, constructor.body_text) - def WriteStateDestructor(self, writer) -> None: + def writeStateDestructor(self, writer) -> None: destructor = Function( - name=f"~{self.stateClassName}", + name=f"~{self.state_class_name}", return_type="", formal_parameters=[], ) - destructor.endIsUnreachable = True - destructor.publicName = True - destructor.indent(self.codeIndent) + destructor.end_is_unreachable = True + destructor.public_name = True + destructor.indent(self.code_indent) destructor.indent(-1) destructor.write_line("{") destructor.indent(+1) - self.ProbeDestroy(destructor, self.actor.name) - self.WriteFunction(writer, destructor, destructor.body_text) + self.probe_destroy(destructor, self.actor.name) + self.writeFunction(writer, destructor, destructor.body_text) - def Compile( + def compile( self, block: CodeBlock, context: Context, ok_to_continue: bool = True ) -> Context: cx = context.clone() cx.next = None for stmt in block.statements: if cx.target is None: - raise ActorCompilerError(stmt.FirstSourceLine, "Unreachable code.") + raise ActorCompilerError(stmt.first_source_line, "Unreachable code.") if cx.next is None: - cx.next = self.getFunction(cx.target.name, "cont", self.loopDepth) + cx.next = self.get_function(cx.target.name, "cont", self.loop_depth) self.compile_statement(stmt, cx) - if cx.next.wasCalled: + if cx.next.was_called: if cx.target is None: raise ActorCompilerError( - stmt.FirstSourceLine, "Unreachable continuation called?" + stmt.first_source_line, "Unreachable continuation called?" ) if not ok_to_continue: raise ActorCompilerError( - stmt.FirstSourceLine, "Unexpected continuation" + stmt.first_source_line, "Unexpected continuation" ) cx.target = cx.next cx.next = None @@ -909,7 +909,7 @@ def compile_statement(self, stmt: Statement, cx: Context) -> None: handler = getattr(self, handler_name, None) if handler is None: raise ActorCompilerError( - stmt.FirstSourceLine, + stmt.first_source_line, "Statement type {0} not supported yet.", stmt.__class__.__name__, ) @@ -926,7 +926,7 @@ def _is_top_level_state_decl(self, stmt: StateDeclarationStatement) -> bool: def _compile_PlainOldCodeStatement( self, stmt: PlainOldCodeStatement, cx: Context ) -> None: - self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + self.line_numberFunction(cx.target, stmt.first_source_line) cx.target.write_line(stmt.code) def _compile_StateDeclarationStatement( @@ -935,26 +935,26 @@ def _compile_StateDeclarationStatement( if self._is_top_level_state_decl(stmt): self.state.append( StateVar( - SourceLine=stmt.FirstSourceLine, + source_line=stmt.first_source_line, name=stmt.decl.name, type=stmt.decl.type, initializer=stmt.decl.initializer, - initializerConstructorSyntax=stmt.decl.initializerConstructorSyntax, + initializer_constructor_syntax=stmt.decl.initializer_constructor_syntax, ) ) else: self.state.append( StateVar( - SourceLine=stmt.FirstSourceLine, + source_line=stmt.first_source_line, name=stmt.decl.name, type=stmt.decl.type, initializer=None, ) ) if stmt.decl.initializer is not None: - self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + self.line_numberFunction(cx.target, stmt.first_source_line) if ( - stmt.decl.initializerConstructorSyntax + stmt.decl.initializer_constructor_syntax or stmt.decl.initializer == "" ): cx.target.write_line( @@ -968,37 +968,37 @@ def _compile_StateDeclarationStatement( "{0} = {1};", stmt.decl.name, stmt.decl.initializer ) - def getIteratorName(self, cx: Context) -> str: + def get_iterator_name(self, cx: Context) -> str: name = f"RangeFor{cx.target.name}Iterator" self.iterators.setdefault(name, 0) idx = self.iterators[name] self.iterators[name] += 1 return f"{name}{idx}" - def EmitNativeLoop( + def emit_native_loop( self, source_line: int, head: str, body: Statement, cx: Context ) -> bool: - self.LineNumberFunction(cx.target, source_line) + self.line_numberFunction(cx.target, source_line) cx.target.write_line(head + " {") cx.target.indent(+1) literal_break = LiteralBreak() literal_continue = LiteralContinue() - self.Compile( - self.AsCodeBlock(body), + self.compile( + self.as_code_block(body), cx.loop_context(cx.target, literal_break, literal_continue, 0), True, ) cx.target.indent(-1) cx.target.write_line("}") - return not literal_break.wasCalled + return not literal_break.was_called def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: - no_condition = stmt.condExpression in ("", "true", "1") - if not self.WillContinue(stmt.body): + no_condition = stmt.cond_expression in ("", "true", "1") + if not self.will_continue(stmt.body): if ( - self.EmitNativeLoop( - stmt.FirstSourceLine, - f"for({stmt.initExpression};{stmt.condExpression};{stmt.nextExpression})", + self.emit_native_loop( + stmt.first_source_line, + f"for({stmt.init_expression};{stmt.cond_expression};{stmt.next_expression})", stmt.body, cx, ) @@ -1008,7 +1008,7 @@ def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: return init_stmt = PlainOldCodeStatement( - code=f"{stmt.initExpression};", FirstSourceLine=stmt.FirstSourceLine + code=f"{stmt.init_expression};", first_source_line=stmt.first_source_line ) self._compile_PlainOldCodeStatement(init_stmt, cx) @@ -1016,22 +1016,22 @@ def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: full_body = stmt.body else: condition = IfStatement( - expression=f"!({stmt.condExpression})", - ifBody=BreakStatement(FirstSourceLine=stmt.FirstSourceLine), - FirstSourceLine=stmt.FirstSourceLine, + expression=f"!({stmt.cond_expression})", + if_body=BreakStatement(first_source_line=stmt.first_source_line), + first_source_line=stmt.first_source_line, ) full_body = CodeBlock( - statements=[condition] + list(self.AsCodeBlock(stmt.body).statements), - FirstSourceLine=stmt.FirstSourceLine, + statements=[condition] + list(self.as_code_block(stmt.body).statements), + first_source_line=stmt.first_source_line, ) - loopF = self.getFunction(cx.target.name, "loopHead", self.loopDepth) - loopBody = self.getFunction(cx.target.name, "loopBody", self.loopDepth) - breakF = self.getFunction(cx.target.name, "break", self.loopDepth) - continueF = ( + loopF = self.get_function(cx.target.name, "loopHead", self.loop_depth) + loopBody = self.get_function(cx.target.name, "loopBody", self.loop_depth) + break_f = self.get_function(cx.target.name, "break", self.loop_depth) + continue_f = ( loopF - if stmt.nextExpression == "" - else self.getFunction(cx.target.name, "continue", self.loopDepth) + if stmt.next_expression == "" + else self.get_function(cx.target.name, "continue", self.loop_depth) ) loopF.write_line("int oldLoopDepth = ++loopDepth;") @@ -1040,18 +1040,18 @@ def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: loopBody.call("loopDepth"), ) - endLoop = self.Compile( - self.AsCodeBlock(full_body), - cx.loop_context(loopBody, breakF, continueF, +1), + endLoop = self.compile( + self.as_code_block(full_body), + cx.loop_context(loopBody, break_f, continue_f, +1), True, ).target if endLoop is not None and endLoop is not loopBody: - if stmt.nextExpression: + if stmt.next_expression: self._compile_PlainOldCodeStatement( PlainOldCodeStatement( - code=f"{stmt.nextExpression};", - FirstSourceLine=stmt.FirstSourceLine, + code=f"{stmt.next_expression};", + first_source_line=stmt.first_source_line, ), cx.with_target(endLoop), ) @@ -1059,115 +1059,115 @@ def _compile_ForStatement(self, stmt: ForStatement, cx: Context) -> None: cx.target.write_line("loopDepth = {0};", loopF.call("loopDepth")) - if continueF is not loopF and continueF.wasCalled: + if continue_f is not loopF and continue_f.was_called: self._compile_PlainOldCodeStatement( PlainOldCodeStatement( - code=f"{stmt.nextExpression};", FirstSourceLine=stmt.FirstSourceLine + code=f"{stmt.next_expression};", first_source_line=stmt.first_source_line ), - cx.with_target(continueF), + cx.with_target(continue_f), ) - continueF.write_line("if (loopDepth == 0) return {0};", loopF.call("0")) - - if breakF.wasCalled: - self.TryCatch( - cx.with_target(breakF), - cx.catchFErr, - cx.tryLoopDepth, - lambda: breakF.write_line("return {0};", cx.next.call("loopDepth")), + continue_f.write_line("if (loopDepth == 0) return {0};", loopF.call("0")) + + if break_f.was_called: + self.try_catch( + cx.with_target(break_f), + cx.catch_f_err, + cx.try_loop_depth, + lambda: break_f.write_line("return {0};", cx.next.call("loopDepth")), ) else: cx.unreachable() def _compile_RangeForStatement(self, stmt: RangeForStatement, cx: Context) -> None: - if self.WillContinue(stmt.body): + if self.will_continue(stmt.body): container = next( - (s for s in self.state if s.name == stmt.rangeExpression), None + (s for s in self.state if s.name == stmt.range_expression), None ) if container is None: raise ActorCompilerError( - stmt.FirstSourceLine, + stmt.first_source_line, "container of range-based for with continuation must be a state variable", ) - iterator_name = self.getIteratorName(cx) + iterator_name = self.get_iterator_name(cx) self.state.append( StateVar( - SourceLine=stmt.FirstSourceLine, + source_line=stmt.first_source_line, name=iterator_name, type=f"decltype(std::begin(std::declval<{container.type}>()))", initializer=None, ) ) equivalent = ForStatement( - initExpression=f"{iterator_name} = std::begin({stmt.rangeExpression})", - condExpression=f"{iterator_name} != std::end({stmt.rangeExpression})", - nextExpression=f"++{iterator_name}", - FirstSourceLine=stmt.FirstSourceLine, + init_expression=f"{iterator_name} = std::begin({stmt.range_expression})", + cond_expression=f"{iterator_name} != std::end({stmt.range_expression})", + next_expression=f"++{iterator_name}", + first_source_line=stmt.first_source_line, body=CodeBlock( statements=[ PlainOldCodeStatement( - code=f"{stmt.rangeDecl} = *{iterator_name};", - FirstSourceLine=stmt.FirstSourceLine, + code=f"{stmt.range_decl} = *{iterator_name};", + first_source_line=stmt.first_source_line, ), stmt.body, ], - FirstSourceLine=stmt.FirstSourceLine, + first_source_line=stmt.first_source_line, ), ) self._compile_ForStatement(equivalent, cx) else: - self.EmitNativeLoop( - stmt.FirstSourceLine, - f"for( {stmt.rangeDecl} : {stmt.rangeExpression} )", + self.emit_native_loop( + stmt.first_source_line, + f"for( {stmt.range_decl} : {stmt.range_expression} )", stmt.body, cx, ) def _compile_WhileStatement(self, stmt: WhileStatement, cx: Context) -> None: equivalent = ForStatement( - initExpression="", - condExpression=stmt.expression, - nextExpression="", + init_expression="", + cond_expression=stmt.expression, + next_expression="", body=stmt.body, - FirstSourceLine=stmt.FirstSourceLine, + first_source_line=stmt.first_source_line, ) self._compile_ForStatement(equivalent, cx) def _compile_LoopStatement(self, stmt: LoopStatement, cx: Context) -> None: equivalent = ForStatement( - initExpression="", - condExpression="", - nextExpression="", + init_expression="", + cond_expression="", + next_expression="", body=stmt.body, - FirstSourceLine=stmt.FirstSourceLine, + first_source_line=stmt.first_source_line, ) self._compile_ForStatement(equivalent, cx) def _compile_BreakStatement(self, stmt: BreakStatement, cx: Context) -> None: - if cx.breakF is None: - raise ActorCompilerError(stmt.FirstSourceLine, "break outside loop") - if isinstance(cx.breakF, LiteralBreak): - cx.target.write_line("{0};", cx.breakF.call()) + if cx.break_f is None: + raise ActorCompilerError(stmt.first_source_line, "break outside loop") + if isinstance(cx.break_f, LiteralBreak): + cx.target.write_line("{0};", cx.break_f.call()) else: cx.target.write_line( - "return {0}; // break", cx.breakF.call("loopDepth==0?0:loopDepth-1") + "return {0}; // break", cx.break_f.call("loopDepth==0?0:loopDepth-1") ) cx.unreachable() def _compile_ContinueStatement(self, stmt: ContinueStatement, cx: Context) -> None: - if cx.continueF is None: - raise ActorCompilerError(stmt.FirstSourceLine, "continue outside loop") - if isinstance(cx.continueF, LiteralContinue): - cx.target.write_line("{0};", cx.continueF.call()) + if cx.continue_f is None: + raise ActorCompilerError(stmt.first_source_line, "continue outside loop") + if isinstance(cx.continue_f, LiteralContinue): + cx.target.write_line("{0};", cx.continue_f.call()) else: cx.target.write_line( - "return {0}; // continue", cx.continueF.call("loopDepth") + "return {0}; // continue", cx.continue_f.call("loopDepth") ) cx.unreachable() def _compile_CodeBlock(self, stmt: CodeBlock, cx: Context) -> None: cx.target.write_line("{") cx.target.indent(+1) - end = self.Compile(stmt, cx, True) + end = self.compile(stmt, cx, True) cx.target.indent(-1) cx.target.write_line("}") if end.target is None: @@ -1176,111 +1176,111 @@ def _compile_CodeBlock(self, stmt: CodeBlock, cx: Context) -> None: end.target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) def _compile_ReturnStatement(self, stmt: ReturnStatement, cx: Context) -> None: - self.LineNumberFunction(cx.target, stmt.FirstSourceLine) - if (stmt.expression == "") != (self.actor.returnType is None): + self.line_numberFunction(cx.target, stmt.first_source_line) + if (stmt.expression == "") != (self.actor.return_type is None): raise ActorCompilerError( - stmt.FirstSourceLine, + stmt.first_source_line, "Return statement does not match actor declaration", ) - if self.actor.returnType is not None: + if self.actor.return_type is not None: if stmt.expression == "Never()": - cx.target.write_line("this->~{0}();", self.stateClassName) - cx.target.write_line("{0}->sendAndDelPromiseRef(Never());", self.This) + cx.target.write_line("this->~{0}();", self.state_class_name) + cx.target.write_line("{0}->sendAndDelPromiseRef(Never());", self.this) else: cx.target.write_line( "if (!{0}->SAV<{1}>::futures) {{ (void)({2}); this->~{3}(); {0}->destroy(); return 0; }}", - self.This, - self.actor.returnType, + self.this, + self.actor.return_type, stmt.expression, - self.stateClassName, + self.state_class_name, ) if any(s.name == stmt.expression for s in self.state): cx.target.write_line( "new (&{0}->SAV< {1} >::value()) {1}(std::move({2})); // state_var_RVO", - self.This, - self.actor.returnType, + self.this, + self.actor.return_type, stmt.expression, ) else: cx.target.write_line( "new (&{0}->SAV< {1} >::value()) {1}({2});", - self.This, - self.actor.returnType, + self.this, + self.actor.return_type, stmt.expression, ) - cx.target.write_line("this->~{0}();", self.stateClassName) - cx.target.write_line("{0}->finishSendAndDelPromiseRef();", self.This) + cx.target.write_line("this->~{0}();", self.state_class_name) + cx.target.write_line("{0}->finishSendAndDelPromiseRef();", self.this) else: - cx.target.write_line("delete {0};", self.This) + cx.target.write_line("delete {0};", self.this) cx.target.write_line("return 0;") cx.unreachable() def _compile_ThrowStatement(self, stmt: ThrowStatement, cx: Context) -> None: - self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + self.line_numberFunction(cx.target, stmt.first_source_line) if stmt.expression == "": - if cx.target.exceptionParameterIs is not None: + if cx.target.exception_parameter_is is not None: cx.target.write_line( "return {0};", - cx.catchFErr.call( - cx.target.exceptionParameterIs, - self.AdjustLoopDepth(cx.tryLoopDepth), + cx.catch_f_err.call( + cx.target.exception_parameter_is, + self.adjust_loop_depth(cx.try_loop_depth), ), ) else: raise ActorCompilerError( - stmt.FirstSourceLine, + stmt.first_source_line, "throw statement with no expression has no current exception in scope", ) else: cx.target.write_line( "return {0};", - cx.catchFErr.call( - stmt.expression, self.AdjustLoopDepth(cx.tryLoopDepth) + cx.catch_f_err.call( + stmt.expression, self.adjust_loop_depth(cx.try_loop_depth) ), ) cx.unreachable() def _compile_IfStatement(self, stmt: IfStatement, cx: Context) -> None: - use_continuation = self.WillContinue(stmt.ifBody) or ( - stmt.elseBody is not None and self.WillContinue(stmt.elseBody) + use_continuation = self.will_continue(stmt.if_body) or ( + stmt.else_body is not None and self.will_continue(stmt.else_body) ) - self.LineNumberFunction(cx.target, stmt.FirstSourceLine) + self.line_numberFunction(cx.target, stmt.first_source_line) constexpr = "constexpr " if stmt.constexpr else "" cx.target.write_line(f"if {constexpr}({stmt.expression})") cx.target.write_line("{") cx.target.indent(+1) - if_target = self.Compile( - self.AsCodeBlock(stmt.ifBody), cx, use_continuation + if_target = self.compile( + self.as_code_block(stmt.if_body), cx, use_continuation ).target if use_continuation and if_target is not None: if_target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) cx.target.indent(-1) cx.target.write_line("}") else_target = None - if stmt.elseBody is not None or use_continuation: + if stmt.else_body is not None or use_continuation: cx.target.write_line("else") cx.target.write_line("{") cx.target.indent(+1) else_target = cx.target - if stmt.elseBody is not None: - else_target = self.Compile( - self.AsCodeBlock(stmt.elseBody), cx, use_continuation + if stmt.else_body is not None: + else_target = self.compile( + self.as_code_block(stmt.else_body), cx, use_continuation ).target if use_continuation and else_target is not None: else_target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) cx.target.indent(-1) cx.target.write_line("}") - if if_target is None and stmt.elseBody is not None and else_target is None: + if if_target is None and stmt.else_body is not None and else_target is None: cx.unreachable() - elif not cx.next.wasCalled and use_continuation: + elif not cx.next.was_called and use_continuation: raise ActorCompilerError( - stmt.FirstSourceLine, "Internal error: IfStatement: next not called?" + stmt.first_source_line, "Internal error: IfStatement: next not called?" ) def _compile_TryStatement(self, stmt: TryStatement, cx: Context) -> None: if len(stmt.catches) != 1: raise ActorCompilerError( - stmt.FirstSourceLine, + stmt.first_source_line, "try statement must have exactly one catch clause", ) reachable = False @@ -1290,25 +1290,25 @@ def _compile_TryStatement(self, stmt: TryStatement, cx: Context) -> None: if catch_expression != "...": if not catch_expression.startswith("Error&"): raise ActorCompilerError( - catch_clause.FirstSourceLine, + catch_clause.first_source_line, "Only type 'Error' or '...' may be caught in an actor function", ) catch_param = catch_expression[6:] if not catch_param: catch_param = "__current_error" - catchFErr = self.getFunction( - cx.target.name, "Catch", f"const Error& {catch_param}", self.loopDepth0 + catch_f_err = self.get_function( + cx.target.name, "Catch", f"const Error& {catch_param}", self.loop_depth_0 ) - catchFErr.exceptionParameterIs = catch_param - end = self.TryCatchCompile( - self.AsCodeBlock(stmt.tryBody), cx.with_catch(catchFErr) + catch_f_err.exception_parameter_is = catch_param + end = self.try_catch_compile( + self.as_code_block(stmt.try_body), cx.with_catch(catch_f_err) ) if end.target is not None: reachable = True - self.TryCatch( + self.try_catch( end, - cx.catchFErr, - cx.tryLoopDepth, + cx.catch_f_err, + cx.try_loop_depth, lambda: end.target.write_line( "loopDepth = {0};", cx.next.call("loopDepth") ), @@ -1316,24 +1316,24 @@ def _compile_TryStatement(self, stmt: TryStatement, cx: Context) -> None: def handle_catch() -> None: nonlocal reachable - cend = self._compile_try_catch_body(catch_clause, catchFErr, cx) + cend = self._compile_try_catch_body(catch_clause, catch_f_err, cx) if cend.target is not None: reachable = True - self.TryCatch( - cx.with_target(catchFErr), - cx.catchFErr, - cx.tryLoopDepth, + self.try_catch( + cx.with_target(catch_f_err), + cx.catch_f_err, + cx.try_loop_depth, handle_catch, ) if not reachable: cx.unreachable() def _compile_try_catch_body( - self, catch_clause: TryStatement.Catch, catchFErr: Function, cx: Context + self, catch_clause: TryStatement.Catch, catch_f_err: Function, cx: Context ) -> Context: - cend = self.Compile( - self.AsCodeBlock(catch_clause.body), cx.with_target(catchFErr), True + cend = self.compile( + self.as_code_block(catch_clause.body), cx.with_target(catch_f_err), True ) if cend.target is not None: cend.target.write_line("loopDepth = {0};", cx.next.call("loopDepth")) @@ -1346,57 +1346,57 @@ def _compile_WaitStatement(self, stmt: WaitStatement, cx: Context) -> None: WhenStatement( wait=stmt, body=None, - FirstSourceLine=stmt.FirstSourceLine, + first_source_line=stmt.first_source_line, ) ], - FirstSourceLine=stmt.FirstSourceLine, + first_source_line=stmt.first_source_line, ), - FirstSourceLine=stmt.FirstSourceLine, + first_source_line=stmt.first_source_line, ) - if not stmt.resultIsState: - cx.next.formalParameters = [ + if not stmt.result_is_state: + cx.next.formal_parameters = [ f"{stmt.result.type} const& {stmt.result.name}", - self.loopDepth, + self.loop_depth, ] cx.next.add_overload( - f"{stmt.result.type} && {stmt.result.name}", self.loopDepth + f"{stmt.result.type} && {stmt.result.name}", self.loop_depth ) self._compile_ChooseStatement(equivalent, cx) def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None: - group = self.chooseGroups + 1 - self.chooseGroups = group + group = self.choose_groups + 1 + self.choose_groups = group codeblock = stmt.body if isinstance(stmt.body, CodeBlock) else None if codeblock is None: raise ActorCompilerError( - stmt.FirstSourceLine, + stmt.first_source_line, "'choose' must be followed by a compound statement.", ) choices = [] for idx, choice_stmt in enumerate(codeblock.statements): if not isinstance(choice_stmt, WhenStatement): raise ActorCompilerError( - choice_stmt.FirstSourceLine, + choice_stmt.first_source_line, "only 'when' statements are valid in a 'choose' block.", ) - index = self.whenCount + idx - param_prefix = "__" if choice_stmt.wait.resultIsState else "" + index = self.when_count + idx + param_prefix = "__" if choice_stmt.wait.result_is_state else "" const_param = f"{choice_stmt.wait.result.type} const& {param_prefix}{choice_stmt.wait.result.name}" rvalue_param = f"{choice_stmt.wait.result.type} && {param_prefix}{choice_stmt.wait.result.name}" - body_func = self.getFunction( + body_func = self.get_function( cx.target.name, "when", - [const_param, self.loopDepth], - overload_formal_parameters=[rvalue_param, self.loopDepth], + [const_param, self.loop_depth], + overload_formal_parameters=[rvalue_param, self.loop_depth], ) future_name = f"__when_expr_{index}" callback_template = ( "ActorSingleCallback" - if choice_stmt.wait.isWaitNext + if choice_stmt.wait.is_wait_next else "ActorCallback" ) - callback_type = f"{callback_template}< {self.fullClassName}, {index}, {choice_stmt.wait.result.type} >" - callback_type_state = f"{callback_template}< {self.className}, {index}, {choice_stmt.wait.result.type} >" + callback_type = f"{callback_template}< {self.full_class_name}, {index}, {choice_stmt.wait.result.type} >" + callback_type_state = f"{callback_template}< {self.class_name}, {index}, {choice_stmt.wait.result.type} >" choices.append( { "Stmt": choice_stmt, @@ -1408,38 +1408,38 @@ def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None: "CallbackTypeState": callback_type_state, } ) - self.whenCount += len(choices) - exit_func = self.getFunction("exitChoose", "", []) - exit_func.returnType = "void" + self.when_count += len(choices) + exit_func = self.get_function("exitChoose", "", []) + exit_func.return_type = "void" exit_func.write_line( - "if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", self.This + "if ({0}->actor_wait_state > 0) {0}->actor_wait_state = 0;", self.this ) for choice in choices: exit_func.write_line( - "{0}->{1}::remove();", self.This, choice["CallbackTypeState"] + "{0}->{1}::remove();", self.this, choice["CallbackTypeState"] ) - exit_func.endIsUnreachable = True + exit_func.end_is_unreachable = True reachable = False for choice in choices: self.callbacks.append( CallbackVar( - SourceLine=choice["Stmt"].FirstSourceLine, - CallbackGroup=choice["Group"], + source_line=choice["Stmt"].first_source_line, + callback_group=choice["Group"], type=choice["CallbackType"], ) ) r = choice["Body"] - if choice["Stmt"].wait.resultIsState: + if choice["Stmt"].wait.result_is_state: overload = r.pop_overload() decl_stmt = StateDeclarationStatement( decl=VarDeclaration( type=choice["Stmt"].wait.result.type, name=choice["Stmt"].wait.result.name, initializer=f"__{choice['Stmt'].wait.result.name}", - initializerConstructorSyntax=False, + initializer_constructor_syntax=False, ), - FirstSourceLine=choice["Stmt"].FirstSourceLine, + first_source_line=choice["Stmt"].first_source_line, ) self._compile_StateDeclarationStatement(decl_stmt, cx.with_target(r)) if overload is not None: @@ -1449,12 +1449,12 @@ def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None: ) r.set_overload(overload) if choice["Stmt"].body is not None: - r = self.Compile( - self.AsCodeBlock(choice["Stmt"].body), cx.with_target(r), True + r = self.compile( + self.as_code_block(choice["Stmt"].body), cx.with_target(r), True ).target if r is not None: reachable = True - if len(cx.next.formalParameters) == 1: + if len(cx.next.formal_parameters) == 1: r.write_line("loopDepth = {0};", cx.next.call("loopDepth")) else: overload = r.pop_overload() @@ -1480,21 +1480,21 @@ def _compile_ChooseStatement(self, stmt: ChooseStatement, cx: Context) -> None: f"{choice['Stmt'].wait.result.type} const& value", ], ) - cb_func.endIsUnreachable = True + cb_func.end_is_unreachable = True cb_func.add_overload( f"{choice['CallbackTypeState']}*", f"{choice['Stmt'].wait.result.type} && value", ) - cb_func.indent(self.codeIndent) - self.ProbeEnter(cb_func, self.actor.name, choice["Index"]) + cb_func.indent(self.code_indent) + self.probe_enter(cb_func, self.actor.name, choice["Index"]) cb_func.write_line("{0};", exit_func.call()) overload_func = cb_func.pop_overload() def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None: - self.TryCatch( + self.try_catch( cx.with_target(target_func), - cx.catchFErr, - cx.tryLoopDepth, + cx.catch_f_err, + cx.try_loop_depth, lambda ch=choice_dict, tf=target_func, expression=expr: tf.write_line( "{0};", ch["Body"].call(expression, "0") ), @@ -1505,7 +1505,7 @@ def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None: if overload_func is not None: fire_body(overload_func, "std::move(value)") cb_func.set_overload(overload_func) - self.ProbeExit(cb_func, self.actor.name, choice["Index"]) + self.probe_exit(cb_func, self.actor.name, choice["Index"]) self.functions[f"{cb_func.name}#{choice['Index']}"] = cb_func err_func = Function( @@ -1513,29 +1513,29 @@ def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None: return_type="void", formal_parameters=[f"{choice['CallbackTypeState']}*", "Error err"], ) - err_func.endIsUnreachable = True - err_func.indent(self.codeIndent) - self.ProbeEnter(err_func, self.actor.name, choice["Index"]) + err_func.end_is_unreachable = True + err_func.indent(self.code_indent) + self.probe_enter(err_func, self.actor.name, choice["Index"]) err_func.write_line("{0};", exit_func.call()) - self.TryCatch( + self.try_catch( cx.with_target(err_func), - cx.catchFErr, - cx.tryLoopDepth, - lambda: err_func.write_line("{0};", cx.catchFErr.call("err", "0")), + cx.catch_f_err, + cx.try_loop_depth, + lambda: err_func.write_line("{0};", cx.catch_f_err.call("err", "0")), False, ) - self.ProbeExit(err_func, self.actor.name, choice["Index"]) + self.probe_exit(err_func, self.actor.name, choice["Index"]) self.functions[f"{err_func.name}#{choice['Index']}"] = err_func first_choice = True for choice in choices: - get_func = "pop" if choice["Stmt"].wait.isWaitNext else "get" - self.LineNumberFunction(cx.target, choice["Stmt"].wait.FirstSourceLine) - if choice["Stmt"].wait.isWaitNext: + get_func = "pop" if choice["Stmt"].wait.is_wait_next else "get" + self.line_numberFunction(cx.target, choice["Stmt"].wait.first_source_line) + if choice["Stmt"].wait.is_wait_next: cx.target.write_line( "auto {0} = {1};", choice["Future"], - choice["Stmt"].wait.futureExpression, + choice["Stmt"].wait.future_expression, ) cx.target.write_line( 'static_assert(std::is_same>::value || std::is_same>::value, "invalid type");', @@ -1547,36 +1547,36 @@ def fire_body(target_func: Function, expr: str, choice_dict=choice) -> None: "StrictFuture<{0}> {1} = {2};", choice["Stmt"].wait.result.type, choice["Future"], - choice["Stmt"].wait.futureExpression, + choice["Stmt"].wait.future_expression, ) if first_choice: first_choice = False - self.LineNumberFunction(cx.target, stmt.FirstSourceLine) - if self.actor.IsCancellable(): + self.line_numberFunction(cx.target, stmt.first_source_line) + if self.actor.is_cancellable(): cx.target.write_line( "if ({1}->actor_wait_state < 0) return {0};", - cx.catchFErr.call( - "actor_cancelled()", self.AdjustLoopDepth(cx.tryLoopDepth) + cx.catch_f_err.call( + "actor_cancelled()", self.adjust_loop_depth(cx.try_loop_depth) ), - self.This, + self.this, ) cx.target.write_line( "if ({0}.isReady()) {{ if ({0}.isError()) return {2}; else return {1}; }};", choice["Future"], choice["Body"].call(f"{choice['Future']}.{get_func}()", "loopDepth"), - cx.catchFErr.call( + cx.catch_f_err.call( f"{choice['Future']}.getError()", - self.AdjustLoopDepth(cx.tryLoopDepth), + self.adjust_loop_depth(cx.try_loop_depth), ), ) - cx.target.write_line("{1}->actor_wait_state = {0};", group, self.This) + cx.target.write_line("{1}->actor_wait_state = {0};", group, self.this) for choice in choices: - self.LineNumberFunction(cx.target, choice["Stmt"].wait.FirstSourceLine) + self.line_numberFunction(cx.target, choice["Stmt"].wait.first_source_line) cx.target.write_line( "{0}.addCallbackAndClear(static_cast<{1}*>({2}));", choice["Future"], choice["CallbackTypeState"], - self.This, + self.this, ) cx.target.write_line("loopDepth = 0;") if not reachable: diff --git a/flow/actorcompiler_py/actor_parser.py b/flow/actorcompiler_py/actor_parser.py new file mode 100644 index 00000000000..f6941f85d43 --- /dev/null +++ b/flow/actorcompiler_py/actor_parser.py @@ -0,0 +1,1126 @@ +from __future__ import annotations + +import io +import re +import sys +from dataclasses import dataclass +from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple + +from .errors import ActorCompilerError +from .actor_compiler import ActorCompiler, DescrCompiler + +from .parse_tree import ( + Actor, + BreakStatement, + ChooseStatement, + CodeBlock, + ContinueStatement, + Declaration, + Descr, + ForStatement, + IfStatement, + LoopStatement, + PlainOldCodeStatement, + RangeForStatement, + ReturnStatement, + StateDeclarationStatement, + Statement, + ThrowStatement, + TryStatement, + VarDeclaration, + WaitStatement, + WhenStatement, + WhileStatement, +) + + +class ErrorMessagePolicy: + def __init__(self) -> None: + self.disable_diagnostics = False + + def handle_actor_without_wait(self, source_file: str, actor: Actor) -> None: + if not self.disable_diagnostics and not actor.is_test_case: + print( + f"{source_file}:{actor.source_line}: warning: ACTOR {actor.name} does not contain a wait() statement", + file=sys.stderr, + ) + + def actors_no_discard_by_default(self) -> bool: + return not self.disable_diagnostics + + +@dataclass +class Token: + value: str + position: int = 0 + source_line: int = 0 + brace_depth: int = 0 + paren_depth: int = 0 + + @property + def is_whitespace(self) -> bool: + return ( + self.value in (" ", "\n", "\r", "\r\n", "\t") + or self.value.startswith("//") + or self.value.startswith("/*") + ) + + def ensure(self, error: str, pred) -> "Token": + if not pred(self): + raise ActorCompilerError(self.source_line, error) + return self + + def get_matching_range_in(self, token_range: "TokenRange") -> "TokenRange": + if self.value == "<": + sub = TokenRange(token_range.tokens, self.position, token_range.end_pos) + gen = AngleBracketParser.not_inside_angle_brackets(sub) + next(gen, None) # skip the "<" + closing = next(gen, None) + if closing is None: + raise ActorCompilerError(self.source_line, "Syntax error: Unmatched <") + return TokenRange(token_range.tokens, self.position + 1, closing.position) + if self.value == "[": + sub = TokenRange(token_range.tokens, self.position, token_range.end_pos) + gen = BracketParser.not_inside_brackets(sub) + next(gen, None) # skip the "[" + closing = next(gen, None) + if closing is None: + raise ActorCompilerError(self.source_line, "Syntax error: Unmatched [") + return TokenRange(token_range.tokens, self.position + 1, closing.position) + + pairs = {"(": ")", ")": "(", "{": "}", "}": "{"} + if self.value not in pairs: + raise RuntimeError("Can't match this token") + pred = ( + (lambda t: t.value != ")" or t.paren_depth != self.paren_depth) + if self.value == "(" + else ( + (lambda t: t.value != "(" or t.paren_depth != self.paren_depth) + if self.value == ")" + else ( + (lambda t: t.value != "}" or t.brace_depth != self.brace_depth) + if self.value == "{" + else (lambda t: t.value != "{" or t.brace_depth != self.brace_depth) + ) + ) + ) + direction = 1 if self.value in ("(", "{") else -1 + if direction == -1: + rng = token_range.sub_range(token_range.begin_pos, self.position).Revtake_while(pred) + if rng.begin_pos == token_range.begin_pos: + raise ActorCompilerError( + self.source_line, f"Syntax error: Unmatched {self.value}" + ) + return rng + rng = token_range.sub_range(self.position + 1, token_range.end_pos).take_while(pred) + if rng.end_pos == token_range.end_pos: + raise ActorCompilerError( + self.source_line, f"Syntax error: Unmatched {self.value}" + ) + return rng + + +class TokenRange: + def __init__(self, tokens: List[Token], begin: int, end: int) -> None: + if begin > end: + raise RuntimeError("Invalid TokenRange") + self.tokens = tokens + self.begin_pos = begin + self.end_pos = end + + def __iter__(self) -> Iterator[Token]: + for i in range(self.begin_pos, self.end_pos): + yield self.tokens[i] + + def first(self, predicate=None) -> Token: + if self.begin_pos == self.end_pos: + raise RuntimeError("Empty TokenRange") + if predicate is None: + return self.tokens[self.begin_pos] + for t in self: + if predicate(t): + return t + raise RuntimeError("Matching token not found") + + def last(self, predicate=None) -> Token: + if self.begin_pos == self.end_pos: + raise RuntimeError("Empty TokenRange") + if predicate is None: + return self.tokens[self.end_pos - 1] + for i in range(self.end_pos - 1, self.begin_pos - 1, -1): + if predicate(self.tokens[i]): + return self.tokens[i] + raise RuntimeError("Matching token not found") + + def skip(self, count: int) -> "TokenRange": + return TokenRange(self.tokens, self.begin_pos + count, self.end_pos) + + def consume(self, value_or_error, predicate=None) -> "TokenRange": + if predicate is None: + self.first().ensure( + f"Expected {value_or_error}", lambda t: t.value == value_or_error + ) + else: + self.first().ensure(value_or_error, predicate) + return self.skip(1) + + def skipWhile(self, predicate) -> "TokenRange": + e = self.begin_pos + while e < self.end_pos and predicate(self.tokens[e]): + e += 1 + return TokenRange(self.tokens, e, self.end_pos) + + def take_while(self, predicate) -> "TokenRange": + e = self.begin_pos + while e < self.end_pos and predicate(self.tokens[e]): + e += 1 + return TokenRange(self.tokens, self.begin_pos, e) + + def Revtake_while(self, predicate) -> "TokenRange": + e = self.end_pos - 1 + while e >= self.begin_pos and predicate(self.tokens[e]): + e -= 1 + return TokenRange(self.tokens, e + 1, self.end_pos) + + def Revskip_while(self, predicate) -> "TokenRange": + e = self.end_pos - 1 + while e >= self.begin_pos and predicate(self.tokens[e]): + e -= 1 + return TokenRange(self.tokens, self.begin_pos, e + 1) + + def sub_range(self, begin: int, end: int) -> "TokenRange": + return TokenRange(self.tokens, begin, end) + + def is_empty(self) -> bool: + return self.begin_pos == self.end_pos + + def length(self) -> int: + return self.end_pos - self.begin_pos + + def all_match(self, predicate) -> bool: + return all(predicate(t) for t in self) + + def any_match(self, predicate) -> bool: + return any(predicate(t) for t in self) + + +class BracketParser: + @staticmethod + def not_inside_brackets(tokens: Iterable[Token]) -> Iterator[Token]: + bracket_depth = 0 + base_pd = None + for tok in tokens: + if base_pd is None: + base_pd = tok.paren_depth + if tok.paren_depth == base_pd and tok.value == "]": + bracket_depth -= 1 + if bracket_depth == 0: + yield tok + if tok.paren_depth == base_pd and tok.value == "[": + bracket_depth += 1 + + +class AngleBracketParser: + @staticmethod + def not_inside_angle_brackets(tokens: Iterable[Token]) -> Iterator[Token]: + angle_depth = 0 + base_pd = None + for tok in tokens: + if base_pd is None: + base_pd = tok.paren_depth + if tok.paren_depth == base_pd and tok.value == ">": + angle_depth -= 1 + if angle_depth == 0: + yield tok + if tok.paren_depth == base_pd and tok.value == "<": + angle_depth += 1 + + +class ActorParser: + token_expressions = [ + r"\{", + r"\}", + r"\(", + r"\)", + r"\[", + r"\]", + r"//[^\n]*", + r"/[*]([*][^/]|[^*])*[*]/", + r"'(\\.|[^\'\n])*'", + r'"(\\.|[^"\n])*"', + r"[a-zA-Z_][a-zA-Z_0-9]*", + r"\r\n", + r"\n", + r"::", + r":", + r"#[a-z]*", + r".", + ] + + identifier_pattern = re.compile(r"^[a-zA-Z_][a-zA-Z_0-9]*$") + + def __init__( + self, + text: str, + source_file: str, + error_message_policy: ErrorMessagePolicy, + generate_probes: bool, + ) -> None: + self.source_file = source_file + self.error_message_policy = error_message_policy + self.generate_probes = generate_probes + self.tokens = [Token(value=t) for t in self.tokenize(text)] + self.line_numbers_enabled = True + self.uid_objects: Dict[Tuple[int, int], str] = {} + self.token_array = self.tokens + self.count_parens() + + def tokenize(self, text: str) -> List[str]: + regexes = [re.compile(pattern, re.S) for pattern in self.token_expressions] + pos = 0 + tokens: List[str] = [] + while pos < len(text): + for regex in regexes: + m = regex.match(text, pos) + if m: + tokens.append(m.group(0)) + pos += len(m.group(0)) + break + else: + raise RuntimeError(f"Can't tokenize! {pos}") + return tokens + + def count_parens(self) -> None: + brace_depth = 0 + paren_depth = 0 + line_count = 1 + last_paren = None + last_brace = None + for i, token in enumerate(self.tokens): + value = token.value + if value == "}": + brace_depth -= 1 + elif value == ")": + paren_depth -= 1 + elif value == "\r\n": + line_count += 1 + elif value == "\n": + line_count += 1 + + if brace_depth < 0: + raise ActorCompilerError(line_count, "Mismatched braces") + if paren_depth < 0: + raise ActorCompilerError(line_count, "Mismatched parenthesis") + + token.position = i + token.source_line = line_count + token.brace_depth = brace_depth + token.paren_depth = paren_depth + + if value.startswith("/*"): + line_count += value.count("\n") + if value == "{": + brace_depth += 1 + if brace_depth == 1: + last_brace = token + elif value == "(": + paren_depth += 1 + if paren_depth == 1: + last_paren = token + if brace_depth != 0: + raise ActorCompilerError( + last_brace.source_line if last_brace else line_count, "Unmatched brace" + ) + if paren_depth != 0: + raise ActorCompilerError( + last_paren.source_line if last_paren else line_count, + "Unmatched parenthesis", + ) + + def write(self, writer: io.TextIOBase, destFileName: str) -> None: + ActorCompiler.used_class_names.clear() + writer.write("#define POST_ACTOR_COMPILER 1\n") + outLine = 1 + if self.line_numbers_enabled: + writer.write(f'#line {self.tokens[0].source_line} "{self.source_file}"\n') + outLine += 1 + inBlocks = 0 + classContextStack: List[Tuple[str, int]] = [] + i = 0 + while i < len(self.tokens): + tok = self.tokens[i] + if tok.source_line == 0: + raise RuntimeError("Invalid source line (0)") + if tok.value in ("ACTOR", "SWIFT_ACTOR", "TEST_CASE"): + actor = self.parse_actor(i) + end = self._parse_end + if classContextStack: + actor.enclosing_class = "::".join( + name for name, _ in classContextStack + ) + actor_writer = io.StringIO() + actorCompiler = ActorCompiler( + actor, + self.source_file, + inBlocks == 0, + self.line_numbers_enabled, + self.generate_probes, + ) + actorCompiler.write(actor_writer) + for key, value in actorCompiler.uid_objects.items(): + self.uid_objects.setdefault(key, value) + actor_lines = actor_writer.getvalue().split("\n") + hasLineNumber = False + hadLineNumber = True + for line in actor_lines: + if self.line_numbers_enabled: + isLine = "#line" in line + if isLine: + hadLineNumber = True + if not isLine and not hasLineNumber and hadLineNumber: + writer.write( + '\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t#line {0} "{1}"\n'.format( + outLine + 1, destFileName + ) + ) + outLine += 1 + hadLineNumber = False + hasLineNumber = isLine + writer.write(line.rstrip("\n\r") + "\n") + outLine += 1 + i = end + if i < len(self.tokens) and self.line_numbers_enabled: + writer.write( + f'#line {self.tokens[i].source_line} "{self.source_file}"\n' + ) + outLine += 1 + elif tok.value == "DESCR": + descr, end = self.parse_descr(i) + descr_writer = io.StringIO() + lines = DescrCompiler(descr, tok.brace_depth).write(descr_writer) + writer.write(descr_writer.getvalue()) + outLine += lines + i = end + if i < len(self.tokens) and self.line_numbers_enabled: + writer.write( + f'#line {self.tokens[i].source_line} "{self.source_file}"\n' + ) + outLine += 1 + elif tok.value in ("class", "struct", "union"): + writer.write(tok.value) + success, name = self.parse_class_context( + self.range(i + 1, len(self.tokens)) + ) + if success: + classContextStack.append((name, inBlocks)) + else: + if tok.value == "{": + inBlocks += 1 + elif tok.value == "}": + inBlocks -= 1 + if classContextStack and classContextStack[-1][1] == inBlocks: + classContextStack.pop() + writer.write(tok.value) + outLine += tok.value.count("\n") + i += 1 + + # Parsing helpers + + def range(self, begin: int, end: int) -> TokenRange: + return TokenRange(self.tokens, begin, end) + + def parse_actor(self, pos: int) -> Actor: + token = self.tokens[pos] + actor = Actor() + actor.source_line = token.source_line + toks = self.range(pos + 1, len(self.tokens)) + heading = toks.take_while(lambda t: t.value != "{") + toSemicolon = toks.take_while(lambda t: t.value != ";") + actor.is_forward_declaration = toSemicolon.length() < heading.length() + if actor.is_forward_declaration: + heading = toSemicolon + if token.value in ("ACTOR", "SWIFT_ACTOR"): + self.parse_actorHeading(actor, heading) + else: + token.ensure("ACTOR expected!", lambda _: False) + self._parse_end = heading.end_pos + 1 + else: + body = self.range(heading.end_pos + 1, len(self.tokens)).take_while( + lambda t: t.brace_depth > toks.first().brace_depth + ) + if token.value in ("ACTOR", "SWIFT_ACTOR"): + self.parse_actorHeading(actor, heading) + elif token.value == "TEST_CASE": + self.parse_test_caseHeading(actor, heading) + actor.is_test_case = True + else: + token.ensure("ACTOR or TEST_CASE expected!", lambda _: False) + actor.body = self.parse_code_block(body) + if not actor.body.contains_wait(): + self.error_message_policy.handle_actor_without_wait(self.source_file, actor) + self._parse_end = body.end_pos + 1 + return actor + + def parse_descr(self, pos: int) -> Tuple[Descr, int]: + descr = Descr() + toks = self.range(pos + 1, len(self.tokens)) + heading = toks.take_while(lambda t: t.value != "{") + body = self.range(heading.end_pos + 1, len(self.tokens)).take_while( + lambda t: t.brace_depth > toks.first().brace_depth or t.value == ";" + ) + self.parse_descrHeading(descr, heading) + descr.body = self.parse_descrCodeBlock(body) + end = body.end_pos + 1 + return descr, end + + def parse_descrHeading(self, descr: Descr, toks: TokenRange) -> None: + nonWhitespace = lambda t: not t.is_whitespace + toks.first(nonWhitespace).ensure( + "non-struct DESCR!", lambda t: t.value == "struct" + ) + toks = ( + toks.skipWhile(lambda t: t.is_whitespace) + .skip(1) + .skipWhile(lambda t: t.is_whitespace) + ) + colon = next((t for t in toks if t.value == ":"), None) + if colon: + descr.super_class_list = self.str( + self.range(colon.position + 1, toks.end_pos) + ).strip() + toks = self.range(toks.begin_pos, colon.position) + descr.name = self.str(toks).strip() + + def parse_descrCodeBlock(self, toks: TokenRange) -> List[Declaration]: + declarations: List[Declaration] = [] + while True: + delim = next((t for t in toks if t.value == ";"), None) + if delim is None: + break + pos = delim.position + 1 + potential_comment = self.range(pos, toks.end_pos).skipWhile( + lambda t: t.value in ("\t", " ") + ) + if ( + not potential_comment.is_empty() + and potential_comment.first().value.startswith("//") + ): + pos = potential_comment.first().position + 1 + self.parse_declarationRange(self.range(toks.begin_pos, pos), declarations) + toks = self.range(pos, toks.end_pos) + if not toks.all_match(lambda t: t.is_whitespace): + raise ActorCompilerError( + toks.first(lambda t: not t.is_whitespace).source_line, + "Trailing unterminated statement in code block", + ) + return declarations + + def parse_declarationRange( + self, toks: TokenRange, declarations: List[Declaration] + ) -> None: + delim = toks.first(lambda t: t.value == ";") + nameRange = ( + self.range(toks.begin_pos, delim.position) + .Revskip_while(lambda t: t.is_whitespace) + .Revtake_while(lambda t: not t.is_whitespace) + ) + typeRange = self.range(toks.begin_pos, nameRange.begin_pos) + commentRange = self.range(delim.position + 1, toks.end_pos) + declarations.append( + Declaration( + name=self.str(nameRange).strip(), + type=self.str(typeRange).strip(), + comment=self.str(commentRange).strip().lstrip("/"), + ) + ) + + def parse_test_caseHeading(self, actor: Actor, toks: TokenRange) -> None: + actor.is_static = True + nonWhitespace = lambda t: not t.is_whitespace + paramRange = ( + toks.last(nonWhitespace) + .ensure( + "Unexpected tokens after test case parameter list.", + lambda t: t.value == ")" and t.paren_depth == toks.first().paren_depth, + ) + .get_matching_range_in(toks) + ) + actor.test_case_parameters = self.str(paramRange) + actor.name = f"flowTestCase{toks.first().source_line}" + actor.parameters = [ + VarDeclaration( + name="params", + type="UnitTestParameters", + initializer="", + initializer_constructor_syntax=False, + ) + ] + actor.return_type = "Void" + + def parse_actorHeading(self, actor: Actor, toks: TokenRange) -> None: + nonWhitespace = lambda t: not t.is_whitespace + template = toks.first(nonWhitespace) + if template.value == "template": + templateParams = ( + self.range(template.position + 1, toks.end_pos) + .first(nonWhitespace) + .ensure("Invalid template declaration", lambda t: t.value == "<") + .get_matching_range_in(toks) + ) + actor.template_formals = [ + self.parse_var_declaration(p) + for p in self.split_parameter_list(templateParams, ",") + ] + toks = self.range(templateParams.end_pos + 1, toks.end_pos) + attribute = toks.first(nonWhitespace) + while attribute.value == "[": + contents = attribute.get_matching_range_in(toks) + as_array = list(contents) + if ( + len(as_array) < 2 + or as_array[0].value != "[" + or as_array[-1].value != "]" + ): + raise ActorCompilerError( + actor.source_line, "Invalid attribute: Expected [[...]]" + ) + actor.attributes.append( + "[" + self.str(self.normalize_whitespace(contents)) + "]" + ) + toks = self.range(contents.end_pos + 1, toks.end_pos) + attribute = toks.first(nonWhitespace) + static_keyword = toks.first(nonWhitespace) + if static_keyword.value == "static": + actor.is_static = True + toks = self.range(static_keyword.position + 1, toks.end_pos) + uncancellable = toks.first(nonWhitespace) + if uncancellable.value == "UNCANCELLABLE": + actor.set_uncancellable() + toks = self.range(uncancellable.position + 1, toks.end_pos) + paramRange = ( + toks.last(nonWhitespace) + .ensure( + "Unexpected tokens after actor parameter list.", + lambda t: t.value == ")" and t.paren_depth == toks.first().paren_depth, + ) + .get_matching_range_in(toks) + ) + actor.parameters = [ + self.parse_var_declaration(p) + for p in self.split_parameter_list(paramRange, ",") + ] + nameToken = self.range(toks.begin_pos, paramRange.begin_pos - 1).last(nonWhitespace) + actor.name = nameToken.value + return_range = self.range( + toks.first().position + 1, nameToken.position + ).skipWhile(lambda t: t.is_whitespace) + retToken = return_range.first() + if retToken.value == "Future": + ofType = ( + return_range.skip(1) + .first(nonWhitespace) + .ensure("Expected <", lambda tok: tok.value == "<") + .get_matching_range_in(return_range) + ) + actor.return_type = self.str(self.normalize_whitespace(ofType)) + toks = self.range(ofType.end_pos + 1, return_range.end_pos) + elif retToken.value == "void": + actor.return_type = None + toks = return_range.skip(1) + else: + raise ActorCompilerError( + actor.source_line, "Actor apparently does not return Future" + ) + toks = toks.skipWhile(lambda t: t.is_whitespace) + if not toks.is_empty(): + if toks.last().value == "::": + actor.name_space = self.str(self.range(toks.begin_pos, toks.end_pos - 1)) + else: + raise ActorCompilerError( + actor.source_line, + "Unrecognized tokens preceding parameter list in actor declaration", + ) + if ( + self.error_message_policy.actors_no_discard_by_default() + and "[[flow_allow_discard]]" not in actor.attributes + ): + if actor.is_cancellable(): + actor.attributes.append("[[nodiscard]]") + known_flow_attributes = {"[[flow_allow_discard]]"} + for flow_attribute in [a for a in actor.attributes if a.startswith("[[flow_")]: + if flow_attribute not in known_flow_attributes: + raise ActorCompilerError( + actor.source_line, f"Unknown flow attribute {flow_attribute}" + ) + actor.attributes = [a for a in actor.attributes if not a.startswith("[[flow_")] + + def parse_var_declaration(self, tokens: TokenRange) -> VarDeclaration: + name, typeRange, initializer, constructorSyntax = self.parse_declaration(tokens) + return VarDeclaration( + name=name.value, + type=self.str(self.normalize_whitespace(typeRange)), + initializer=( + "" + if initializer is None + else self.str(self.normalize_whitespace(initializer)) + ), + initializer_constructor_syntax=constructorSyntax, + ) + + def parse_declaration(self, tokens: TokenRange): + nonWhitespace = lambda t: not t.is_whitespace + initializer = None + beforeInitializer = tokens + constructorSyntax = False + equals = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(tokens) + if t.value == "=" and t.paren_depth == tokens.first().paren_depth + ), + None, + ) + if equals: + beforeInitializer = self.range(tokens.begin_pos, equals.position) + initializer = self.range(equals.position + 1, tokens.end_pos) + else: + paren = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(tokens) + if t.value == "(" + ), + None, + ) + if paren: + constructorSyntax = True + beforeInitializer = self.range(tokens.begin_pos, paren.position) + initializer = self.range(paren.position + 1, tokens.end_pos).take_while( + lambda t, p=paren.paren_depth: t.paren_depth > p + ) + else: + brace = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(tokens) + if t.value == "{" + ), + None, + ) + if brace: + raise ActorCompilerError( + brace.source_line, + "Uniform initialization syntax is not currently supported for state variables (use '(' instead of '}' ?)", + ) + name = beforeInitializer.last(nonWhitespace) + if beforeInitializer.begin_pos == name.position: + raise ActorCompilerError( + beforeInitializer.first().source_line, "Declaration has no type." + ) + typeRange = self.range(beforeInitializer.begin_pos, name.position) + return name, typeRange, initializer, constructorSyntax + + def normalize_whitespace(self, tokens: Iterable[Token]) -> Iterable[Token]: + inWhitespace = False + leading = True + for tok in tokens: + if not tok.is_whitespace: + if inWhitespace and not leading: + yield Token(value=" ") + inWhitespace = False + yield tok + leading = False + else: + inWhitespace = True + + def split_parameter_list( + self, toks: TokenRange, delimiter: str + ) -> Iterable[TokenRange]: + if toks.begin_pos == toks.end_pos: + return [] + ranges: List[TokenRange] = [] + while True: + comma = next( + ( + t + for t in AngleBracketParser.not_inside_angle_brackets(toks) + if t.value == delimiter and t.paren_depth == toks.first().paren_depth + ), + None, + ) + if comma is None: + break + ranges.append(self.range(toks.begin_pos, comma.position)) + toks = self.range(comma.position + 1, toks.end_pos) + ranges.append(toks) + return ranges + + def parse_loop_statement(self, toks: TokenRange) -> LoopStatement: + return LoopStatement(body=self.parse_compound_statement(toks.consume("loop"))) + + def parse_choose_statement(self, toks: TokenRange) -> ChooseStatement: + return ChooseStatement(body=self.parse_compound_statement(toks.consume("choose"))) + + def parse_when_statement(self, toks: TokenRange) -> WhenStatement: + expr = ( + toks.consume("when") + .skipWhile(lambda t: t.is_whitespace) + .first() + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + .skipWhile(lambda t: t.is_whitespace) + ) + return WhenStatement( + wait=self.parse_wait_statement(expr), + body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + ) + + def parse_state_declaration(self, toks: TokenRange) -> StateDeclarationStatement: + toks = toks.consume("state").Revskip_while(lambda t: t.value == ";") + return StateDeclarationStatement(decl=self.parse_var_declaration(toks)) + + def parse_return_statement(self, toks: TokenRange) -> ReturnStatement: + toks = toks.consume("return").Revskip_while(lambda t: t.value == ";") + return ReturnStatement(expression=self.str(self.normalize_whitespace(toks))) + + def parse_throw_statement(self, toks: TokenRange) -> ThrowStatement: + toks = toks.consume("throw").Revskip_while(lambda t: t.value == ";") + return ThrowStatement(expression=self.str(self.normalize_whitespace(toks))) + + def parse_wait_statement(self, toks: TokenRange) -> WaitStatement: + ws = WaitStatement() + ws.first_source_line = toks.first().source_line + if toks.first().value == "state": + ws.result_is_state = True + toks = toks.consume("state") + initializer = None + if toks.first().value in ("wait", "waitNext"): + initializer = toks.Revskip_while(lambda t: t.value == ";") + ws.result = VarDeclaration( + name="_", + type="Void", + initializer="", + initializer_constructor_syntax=False, + ) + else: + name, typeRange, initializer, constructorSyntax = self.parse_declaration( + toks.Revskip_while(lambda t: t.value == ";") + ) + type_str = self.str(self.normalize_whitespace(typeRange)) + if type_str == "Void": + raise ActorCompilerError( + ws.first_source_line, + "Assigning the result of a Void wait is not allowed. Just use a standalone wait statement.", + ) + ws.result = VarDeclaration( + name=name.value, + type=self.str(self.normalize_whitespace(typeRange)), + initializer="", + initializer_constructor_syntax=False, + ) + if initializer is None: + raise ActorCompilerError( + ws.first_source_line, + "Wait statement must be a declaration or standalone statement", + ) + waitParams = ( + initializer.skipWhile(lambda t: t.is_whitespace) + .consume( + "Statement contains a wait, but is not a valid wait statement or a supported compound statement.1", + lambda t: True if t.value in ("wait", "waitNext") else False, + ) + .skipWhile(lambda t: t.is_whitespace) + .first() + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(initializer) + ) + if ( + not self.range(waitParams.end_pos, initializer.end_pos) + .consume(")") + .all_match(lambda t: t.is_whitespace) + ): + raise ActorCompilerError( + toks.first().source_line, + "Statement contains a wait, but is not a valid wait statement or a supported compound statement.2", + ) + ws.future_expression = self.str(self.normalize_whitespace(waitParams)) + ws.is_wait_next = "waitNext" in [t.value for t in initializer] + return ws + + def parse_while_statement(self, toks: TokenRange) -> WhileStatement: + expr = ( + toks.consume("while") + .first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + return WhileStatement( + expression=self.str(self.normalize_whitespace(expr)), + body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + ) + + def parse_for_statement(self, toks: TokenRange) -> Statement: + head = ( + toks.consume("for") + .first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + delim = [ + t + for t in head + if t.paren_depth == head.first().paren_depth + and t.brace_depth == head.first().brace_depth + and t.value == ";" + ] + if len(delim) == 2: + init = self.range(head.begin_pos, delim[0].position) + cond = self.range(delim[0].position + 1, delim[1].position) + next_expr = self.range(delim[1].position + 1, head.end_pos) + body = self.range(head.end_pos + 1, toks.end_pos) + return ForStatement( + init_expression=self.str(self.normalize_whitespace(init)), + cond_expression=self.str(self.normalize_whitespace(cond)), + next_expression=self.str(self.normalize_whitespace(next_expr)), + body=self.parse_compound_statement(body), + ) + delim = [ + t + for t in head + if t.paren_depth == head.first().paren_depth + and t.brace_depth == head.first().brace_depth + and t.value == ":" + ] + if len(delim) != 1: + raise ActorCompilerError( + head.first().source_line, + "for statement must be 3-arg style or c++11 2-arg style", + ) + return RangeForStatement( + range_expression=self.str( + self.normalize_whitespace( + self.range(delim[0].position + 1, head.end_pos).skipWhile( + lambda t: t.is_whitespace + ) + ) + ), + range_decl=self.str( + self.normalize_whitespace( + self.range(head.begin_pos, delim[0].position - 1).skipWhile( + lambda t: t.is_whitespace + ) + ) + ), + body=self.parse_compound_statement(self.range(head.end_pos + 1, toks.end_pos)), + ) + + def parse_if_statement(self, toks: TokenRange) -> IfStatement: + toks = toks.consume("if").skipWhile(lambda t: t.is_whitespace) + constexpr = toks.first().value == "constexpr" + if constexpr: + toks = toks.consume("constexpr").skipWhile(lambda t: t.is_whitespace) + expr = ( + toks.first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + return IfStatement( + expression=self.str(self.normalize_whitespace(expr)), + constexpr=constexpr, + if_body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + ) + + def parse_else_statement(self, toks: TokenRange, prev: Statement) -> None: + if_stmt = prev + while isinstance(if_stmt, IfStatement) and if_stmt.else_body is not None: + if_stmt = if_stmt.else_body + if not isinstance(if_stmt, IfStatement): + raise ActorCompilerError( + toks.first().source_line, "else without matching if" + ) + if_stmt.else_body = self.parse_compound_statement(toks.consume("else")) + + def parse_try_statement(self, toks: TokenRange) -> TryStatement: + return TryStatement( + try_body=self.parse_compound_statement(toks.consume("try")), + catches=[], + ) + + def parse_catch_statement(self, toks: TokenRange, prev: Statement) -> None: + if not isinstance(prev, TryStatement): + raise ActorCompilerError( + toks.first().source_line, "catch without matching try" + ) + expr = ( + toks.consume("catch") + .first(lambda t: not t.is_whitespace) + .ensure("Expected (", lambda t: t.value == "(") + .get_matching_range_in(toks) + ) + prev.catches.append( + TryStatement.Catch( + expression=self.str(self.normalize_whitespace(expr)), + body=self.parse_compound_statement(self.range(expr.end_pos + 1, toks.end_pos)), + first_source_line=expr.first().source_line, + ) + ) + + illegal_keywords = {"goto", "do", "finally", "__if_exists", "__if_not_exists"} + + def parse_compound_statement(self, toks: TokenRange) -> Statement: + nonWhitespace = lambda t: not t.is_whitespace + first = toks.first(nonWhitespace) + if first.value == "{": + inBraces = first.get_matching_range_in(toks) + if ( + not self.range(inBraces.end_pos, toks.end_pos) + .consume("}") + .all_match(lambda t: t.is_whitespace) + ): + raise ActorCompilerError( + inBraces.last().source_line, + "Unexpected tokens after compound statement", + ) + return self.parse_code_block(inBraces) + statements: List[Statement] = [] + self.parse_statement(toks.skip(1), statements) + return statements[0] + + def parse_code_block(self, toks: TokenRange) -> CodeBlock: + statements: List[Statement] = [] + while True: + delim = next( + ( + t + for t in toks + if t.paren_depth == toks.first().paren_depth + and t.brace_depth == toks.first().brace_depth + and t.value in (";", "}") + ), + None, + ) + if delim is None: + break + self.parse_statement(self.range(toks.begin_pos, delim.position + 1), statements) + toks = self.range(delim.position + 1, toks.end_pos) + if not toks.all_match(lambda t: t.is_whitespace): + raise ActorCompilerError( + toks.first(lambda t: not t.is_whitespace).source_line, + "Trailing unterminated statement in code block", + ) + return CodeBlock(statements=statements) + + def parse_statement(self, toks: TokenRange, statements: List[Statement]) -> None: + nonWhitespace = lambda t: not t.is_whitespace + toks = toks.skipWhile(lambda t: t.is_whitespace) + + def Add(stmt: Statement) -> None: + stmt.first_source_line = toks.first().source_line + statements.append(stmt) + + first_val = toks.first().value + if first_val == "loop": + Add(self.parse_loop_statement(toks)) + elif first_val == "while": + Add(self.parse_while_statement(toks)) + elif first_val == "for": + Add(self.parse_for_statement(toks)) + elif first_val == "break": + Add(BreakStatement()) + elif first_val == "continue": + Add(ContinueStatement()) + elif first_val == "return": + Add(self.parse_return_statement(toks)) + elif first_val == "{": + Add(self.parse_compound_statement(toks)) + elif first_val == "if": + Add(self.parse_if_statement(toks)) + elif first_val == "else": + self.parse_else_statement(toks, statements[-1]) + elif first_val == "choose": + Add(self.parse_choose_statement(toks)) + elif first_val == "when": + Add(self.parse_when_statement(toks)) + elif first_val == "try": + Add(self.parse_try_statement(toks)) + elif first_val == "catch": + self.parse_catch_statement(toks, statements[-1]) + elif first_val == "throw": + Add(self.parse_throw_statement(toks)) + else: + if first_val in self.illegal_keywords: + raise ActorCompilerError( + toks.first().source_line, + f"Statement '{first_val}' not supported in actors.", + ) + if any(t.value in ("wait", "waitNext") for t in toks): + Add(self.parse_wait_statement(toks)) + elif first_val == "state": + Add(self.parse_state_declaration(toks)) + elif first_val == "switch" and any(t.value == "return" for t in toks): + raise ActorCompilerError( + toks.first().source_line, + "Unsupported compound statement containing return.", + ) + elif first_val.startswith("#"): + raise ActorCompilerError( + toks.first().source_line, + f'Found "{first_val}". Preprocessor directives are not supported within ACTORs', + ) + else: + cleaned = toks.Revskip_while(lambda t: t.value == ";") + if any(nonWhitespace(t) for t in cleaned): + Add( + PlainOldCodeStatement( + code=self.str(self.normalize_whitespace(cleaned)) + ";" + ) + ) + + def parse_class_context(self, toks: TokenRange) -> Tuple[bool, str]: + name = "" + if toks.begin_pos == toks.end_pos: + return False, name + + nonWhitespace = lambda t: not t.is_whitespace + while True: + first = toks.first(nonWhitespace) + if first.value == "[": + contents = first.get_matching_range_in(toks) + toks = self.range(contents.end_pos + 1, toks.end_pos) + elif first.value == "alignas": + toks = self.range(first.position + 1, toks.end_pos) + first = toks.first(nonWhitespace) + first.ensure("Expected ( after alignas", lambda t: t.value == "(") + contents = first.get_matching_range_in(toks) + toks = self.range(contents.end_pos + 1, toks.end_pos) + else: + break + + first = toks.first(nonWhitespace) + if not self.identifier_pattern.match(first.value): + return False, name + + while True: + first.ensure( + "Expected identifier", lambda t: self.identifier_pattern.match(t.value) + ) + name += first.value + toks = self.range(first.position + 1, toks.end_pos) + next_token = toks.first(nonWhitespace) + if next_token.value == "::": + name += "::" + toks = toks.skipWhile(lambda t: t.is_whitespace).skip(1) + else: + break + first = toks.first(nonWhitespace) + + toks = toks.skipWhile( + lambda t: t.is_whitespace or t.value in ("final", "explicit") + ) + first = toks.first(nonWhitespace) + if first.value in (":", "{"): + return True, name + return False, "" + + def str(self, tokens: Iterable[Token]) -> str: + return "".join(tok.value for tok in tokens) diff --git a/flow/actorcompiler_py/compare_actor_output.py b/flow/actorcompiler_py/compare_actor_output.py new file mode 100644 index 00000000000..e4208ebb4c4 --- /dev/null +++ b/flow/actorcompiler_py/compare_actor_output.py @@ -0,0 +1,60 @@ +import argparse +import sys +import re +import difflib +from pathlib import Path + +def compare_outputs(file1_path: Path, file2_path: Path) -> bool: + try: + file1_content = file1_path.read_text() + file2_content = file2_path.read_text() + + if file1_content == file2_content: + return True + + # Normalize #line directives - replace the filename part with a constant + # Pattern: #line "filename" + line_directive_pattern = re.compile(r'(#line\s+\d+\s+")[^"]*(")') + + file1_normalized = line_directive_pattern.sub( + r"\1NORMALIZED_PATH\2", file1_content + ) + file2_normalized = line_directive_pattern.sub( + r"\1NORMALIZED_PATH\2", file2_content + ) + + if file1_normalized == file2_normalized: + return True + + # Generate detailed diff of normalized content + file1_lines = file1_normalized.splitlines(keepends=True) + file2_lines = file2_normalized.splitlines(keepends=True) + + diff = difflib.unified_diff( + file1_lines, + file2_lines, + fromfile=f"{file1_path} (normalized)", + tofile=f"{file2_path} (normalized)", + lineterm="", + ) + + diff_text = "".join(diff) + print(diff_text, file=sys.stderr) + return False + + except Exception as e: + print(f"Error comparing files: {e}", file=sys.stderr) + return False + +def main(): + parser = argparse.ArgumentParser(description="Compare actor compiler outputs") + parser.add_argument("file1", type=Path, help="First file to compare") + parser.add_argument("file2", type=Path, help="Second file to compare") + args = parser.parse_args() + + if not compare_outputs(args.file1, args.file2): + sys.exit(1) + +if __name__ == "__main__": + main() + diff --git a/flow/actorcompiler/__init__.py b/flow/actorcompiler_py/errors.py similarity index 51% rename from flow/actorcompiler/__init__.py rename to flow/actorcompiler_py/errors.py index 71a1db4a044..6ad783a7246 100644 --- a/flow/actorcompiler/__init__.py +++ b/flow/actorcompiler_py/errors.py @@ -1,10 +1,4 @@ -""" -Python port of the Flow actor compiler. - -Typical usage: - from flow.actorcompiler import main - main(["input.actor.cpp", "output.g.actor.cpp"]) -""" +from __future__ import annotations class ActorCompilerError(Exception): @@ -19,15 +13,3 @@ def __init__(self, source_line: int, message: str, *args: object) -> None: def __str__(self) -> str: return f"{super().__str__()} (line {self.source_line})" - -from .actor_compiler import ActorCompiler -from .actor_parser import ActorParser, ErrorMessagePolicy -from .main import main - -__all__ = [ - "ActorCompiler", - "ActorParser", - "ErrorMessagePolicy", - "ActorCompilerError", - "main", -] diff --git a/flow/actorcompiler_py/parse_tree.py b/flow/actorcompiler_py/parse_tree.py new file mode 100644 index 00000000000..18d3cee0a11 --- /dev/null +++ b/flow/actorcompiler_py/parse_tree.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import List, Optional, Sequence + + +@dataclass +class VarDeclaration: + type: str = "" + name: str = "" + initializer: Optional[str] = "" + initializer_constructor_syntax: bool = False + + +@dataclass +class Statement(ABC): + first_source_line: int = 0 + + @abstractmethod + def contains_wait(self) -> bool: + pass + + +@dataclass +class CodeBlock(Statement): + statements: Sequence[Statement] = field(default_factory=list) + + def __str__(self) -> str: + joined = "\n".join(str(stmt) for stmt in self.statements) + return f"CodeBlock\n{joined}\nEndCodeBlock" + + def contains_wait(self) -> bool: + return any(stmt.contains_wait() for stmt in self.statements) + + +@dataclass +class PlainOldCodeStatement(Statement): + code: str = "" + + def __str__(self) -> str: + return self.code + + def contains_wait(self) -> bool: + return False + + +@dataclass +class StateDeclarationStatement(Statement): + decl: VarDeclaration = field(default_factory=VarDeclaration) + + def __str__(self) -> str: + if self.decl.initializer_constructor_syntax: + return f"State {self.decl.type} {self.decl.name}({self.decl.initializer});" + return f"State {self.decl.type} {self.decl.name} = {self.decl.initializer};" + + def contains_wait(self) -> bool: + return False + + +@dataclass +class WhileStatement(Statement): + expression: str = "" + body: Statement = field(default_factory=CodeBlock) + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class ForStatement(Statement): + init_expression: str = "" + cond_expression: str = "" + next_expression: str = "" + body: Statement = field(default_factory=CodeBlock) + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class RangeForStatement(Statement): + range_expression: str = "" + range_decl: str = "" + body: Statement = field(default_factory=CodeBlock) + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class LoopStatement(Statement): + body: Statement = field(default_factory=CodeBlock) + + def __str__(self) -> str: + return f"Loop {self.body}" + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class BreakStatement(Statement): + def contains_wait(self) -> bool: + return False + + +@dataclass +class ContinueStatement(Statement): + def contains_wait(self) -> bool: + return False + + +@dataclass +class IfStatement(Statement): + expression: str = "" + constexpr: bool = False + if_body: Statement = field(default_factory=CodeBlock) + else_body: Optional[Statement] = None + + def contains_wait(self) -> bool: + return self.if_body.contains_wait() or ( + self.else_body is not None and self.else_body.contains_wait() + ) + + +@dataclass +class ReturnStatement(Statement): + expression: str = "" + + def __str__(self) -> str: + return f"Return {self.expression}" + + def contains_wait(self) -> bool: + return False + + +@dataclass +class WaitStatement(Statement): + result: VarDeclaration = field(default_factory=VarDeclaration) + future_expression: str = "" + result_is_state: bool = False + is_wait_next: bool = False + + def __str__(self) -> str: + return f"Wait {self.result.type} {self.result.name} <- {self.future_expression} ({'state' if self.result_is_state else 'local'})" + + def contains_wait(self) -> bool: + return True + + +@dataclass +class ChooseStatement(Statement): + body: Statement = field(default_factory=CodeBlock) + + def __str__(self) -> str: + return f"Choose {self.body}" + + def contains_wait(self) -> bool: + return self.body.contains_wait() + + +@dataclass +class WhenStatement(Statement): + wait: WaitStatement = field(default_factory=WaitStatement) + body: Optional[Statement] = None + + def __str__(self) -> str: + return f"When ({self.wait}) {self.body}" + + def contains_wait(self) -> bool: + return True + + +@dataclass +class TryStatement(Statement): + @dataclass + class Catch: + expression: str = "" + body: Statement = field(default_factory=CodeBlock) + first_source_line: int = 0 + + try_body: Statement = field(default_factory=CodeBlock) + catches: List["TryStatement.Catch"] = field(default_factory=list) + + def contains_wait(self) -> bool: + if self.try_body.contains_wait(): + return True + return any(c.body.contains_wait() for c in self.catches) + + +@dataclass +class ThrowStatement(Statement): + expression: str = "" + + def contains_wait(self) -> bool: + return False + + +@dataclass +class Declaration: + type: str = "" + name: str = "" + comment: str = "" + + +@dataclass +class Actor: + attributes: List[str] = field(default_factory=list) + return_type: Optional[str] = None + name: str = "" + enclosing_class: Optional[str] = None + parameters: Sequence[VarDeclaration] = field(default_factory=list) + template_formals: Optional[Sequence[VarDeclaration]] = None + body: CodeBlock = field(default_factory=CodeBlock) + source_line: int = 0 + is_static: bool = False + _is_uncancellable: bool = False + test_case_parameters: Optional[str] = None + name_space: Optional[str] = None + is_forward_declaration: bool = False + is_test_case: bool = False + + def is_cancellable(self) -> bool: + return self.return_type is not None and not self._is_uncancellable + + def set_uncancellable(self) -> None: + self._is_uncancellable = True + + +@dataclass +class Descr: + name: str = "" + super_class_list: Optional[str] = None + body: List[Declaration] = field(default_factory=list)