Feature: use explicit modeling of continuation on select architecture…

…s/platforms

We use the new explicit modeling of continuations on MacOS, and in ELF binaries for zSeries and ARM/ARM thumb. The results are excellent.

This fixes #573, #929. It also improves #1257.

src/Decompiler/Analysis/ValuePropagator.cs

@@ -156,6 +156,10 @@ public bool Transform(Statement stm)
                     // this statement and the callee.
                     callGraph.AddEdge(stmCur, procCallee);
                 }
+            } else if (ci.Callee is Identifier id &&
+                id == ssa.Procedure.Frame.Continuation)
+            {
+                return ReplaceCallToContinuationWithReturn();
             }
             foreach (var use in ci.Uses)
             {
@@ -173,6 +177,20 @@ public bool Transform(Statement stm)
             return (ci, changed);
         }
 
+        private (Instruction, bool) ReplaceCallToContinuationWithReturn()
+        {
+            var stm = stmCur!;
+            var block = stm.Block;
+            var iCall = block.Statements.IndexOf(stm);
+            for (int i = block.Statements.Count - 1; i >= iCall; --i)
+            {
+                ssa.DeleteStatement(block.Statements[i]);
+            }
+            var ret = new ReturnInstruction();
+            block.Statements.Add(stm.Address, ret);
+            return (ret, true);
+        }
+
         public (Instruction, bool) VisitComment(CodeComment comment)
         {
             return (comment, false);

src/Environments/MacOS/Classic/MacOSClassic.cs

@@ -120,6 +120,7 @@ public override int GetBitSizeFromCBasicType(CBasicType cb)
 
         public override void InjectProcedureEntryStatements(Procedure proc, Address addr, CodeEmitter m)
         {
+            m.MStore(proc.Frame.FramePointer, proc.Frame.Continuation);
             var ptrA5World = EnsureA5Pointer();
             var a5 = proc.Frame.EnsureRegister(Registers.a5);
             m.Assign(a5, ptrA5World);

src/Environments/SysV/SysVPlatform.cs

@@ -160,6 +160,12 @@ public override void InjectProcedureEntryStatements(Procedure proc, Address addr
         {
             switch (Architecture.Name)
             {
+            case "arm":
+            case "arm-thumb":
+                m.Assign(
+                    proc.Frame.EnsureRegister(Architecture.GetRegister("lr")!),
+                    proc.Frame.Continuation);
+                break;
             case "mips-be-32":
             case "mips-le-32":
                 // MIPS ELF ABI: r25 is _always_ set to the address of a procedure on entry.
@@ -177,6 +183,10 @@ public override void InjectProcedureEntryStatements(Procedure proc, Address addr
                     0);
                 break;
             case "zSeries":
+                m.Assign(
+                    proc.Frame.EnsureRegister(Architecture.GetRegister("r14")!),
+                    proc.Frame.Continuation);
+
                 // Stack parameters are passed in starting at offset +160 from the 
                 // stack; everything at lower addresses is local to the called procedure's
                 // frame.

src/UnitTests/Decompiler/Analysis/ValuePropagationTests.cs

@@ -1589,5 +1589,38 @@ define SsaProcedureBuilder
 
             AssertStringsEqual(sExpected, m.Ssa);
         }
+
+        [Test]
+        public void VpCallContinuation()
+        {
+            string sExpected =
+            #region Expected
+@"r2: orig: r2
+    def:  r2 = %continuation
+%continuation:%continuation
+    uses: r2 = %continuation
+// SsaProcedureBuilder
+// Return size: 0
+define SsaProcedureBuilder
+SsaProcedureBuilder_entry:
+	// succ:  l1
+l1:
+	r2 = %continuation
+	return
+	// succ:  SsaProcedureBuilder_exit
+SsaProcedureBuilder_exit:
+";
+            #endregion
+
+            var r2 = m.Reg32("r2", 1);
+            var sidCont = m.Ssa.Identifiers.Add(m.Frame.Continuation, null, false);
+            m.Assign(r2, sidCont.Identifier);
+            m.Call(r2, 0);
+            m.Return();
+
+            RunValuePropagator();
+
+            AssertStringsEqual(sExpected, m.Ssa);
+        }
     }
 }