From 4f9629689b1585318d73f596f60088eb9dd99878 Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Fri, 24 Aug 2018 14:28:34 -0700 Subject: [PATCH 1/2] Implement the Timeout option for ENGINE and SCOPE based tests --- .../Cases/AllCPythonCasesManifest.ini | 2 +- .../Cases/CTypesCPythonCasesManifest.ini | 1 + Src/IronPythonTest/Cases/CaseExecuter.cs | 35 ++++++++++++++----- .../Cases/IronPythonCasesManifest.ini | 1 + .../Cases/StandardCPythonCasesManifest.ini | 1 + Src/IronPythonTest/Util/TestManifest.cs | 2 +- 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini b/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini index d66a51983..08eaa046f 100644 --- a/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini @@ -3,6 +3,7 @@ Ignore=false Arguments="$(TEST_FILE)" WorkingDirectory=$(TEST_FILE_DIR) Redirect=false +Timeout=600000 ; 10 minute timeout [AllCPython.test___all__] IsolationLevel=PROCESS @@ -816,7 +817,6 @@ RunCondition='$(OS)' != 'Unix' AND NOT $(IS_NETCOREAPP) Reason=TODO: figure out IsolationLevel=PROCESS Redirect=true -Timeout=300000 [AllCPython.test_sunau] Ignore=true diff --git a/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini b/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini index af208af3d..9c51b3bd4 100644 --- a/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini @@ -5,6 +5,7 @@ WorkingDirectory=$(TEST_FILE_DIR) Redirect=false RunCondition=NOT $(IS_NETCOREAPP) AND NOT $(IS_MACOS) Reason=ctypes is not working on .NET Core yet +Timeout=600000 ; 10 minute timeout [CTypesCPython.test_errno] Ignore=true diff --git a/Src/IronPythonTest/Cases/CaseExecuter.cs b/Src/IronPythonTest/Cases/CaseExecuter.cs index 0b6dcedcc..86fb93b35 100644 --- a/Src/IronPythonTest/Cases/CaseExecuter.cs +++ b/Src/IronPythonTest/Cases/CaseExecuter.cs @@ -3,6 +3,8 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using System.Text.RegularExpressions; using IronPython.Hosting; using IronPython.Runtime; @@ -164,7 +166,7 @@ private int GetEngineTest(TestInfo testcase) { var source = engine.CreateScriptSourceFromString( testcase.Text, testcase.Path, SourceCodeKind.File); - return GetResult(engine, source, testcase.Path, testcase.Options.WorkingDirectory); + return GetResult(testcase, engine, source, testcase.Path, testcase.Options.WorkingDirectory); } private int GetProcessTest(TestInfo testcase) { @@ -202,7 +204,7 @@ private int GetProcessTest(TestInfo testcase) { if (!proc.WaitForExit(testcase.Options.Timeout)) { proc.Kill(); - Console.Error.Write($"Timed out after {testcase.Options.Timeout / 1000.0} seconds."); + NUnit.Framework.TestContext.Error.WriteLine($"{testcase.Name} timed out after {testcase.Options.Timeout / 1000.0} seconds."); } exitCode = proc.ExitCode; } @@ -232,10 +234,10 @@ private int GetScopeTest(TestInfo testcase) { var source = this.defaultEngine.CreateScriptSourceFromString( testcase.Text, testcase.Path, SourceCodeKind.File); - return GetResult(this.defaultEngine, source, testcase.Path, testcase.Options.WorkingDirectory); + return GetResult(testcase, this.defaultEngine, source, testcase.Path, testcase.Options.WorkingDirectory); } - private int GetResult(ScriptEngine engine, ScriptSource source, string testPath, string workingDir) { + private int GetResult(TestInfo testcase, ScriptEngine engine, ScriptSource source, string testPath, string workingDir) { int res = 0; var path = Environment.GetEnvironmentVariable("IRONPYTHONPATH"); if (string.IsNullOrEmpty(path)) { @@ -255,12 +257,27 @@ private int GetResult(ScriptEngine engine, ScriptSource source, string testPath, var scope = engine.CreateScope(); engine.GetSysModule().SetVariable("argv", List.FromArrayNoCopy(new object[] { source.Path })); var compiledCode = source.Compile(new IronPython.Compiler.PythonCompilerOptions() { ModuleName = "__main__" }); + + var thread = new Thread(() => { + try { + res = engine.Operations.ConvertTo(compiledCode.Execute(scope) ?? 0); + } catch (SystemExitException ex) { + res = ex.GetExitCode(out object otherCode); + } catch (ThreadAbortException) { + Thread.ResetAbort(); + } + }) { + IsBackground = true + }; - try { - res = engine.Operations.ConvertTo(compiledCode.Execute(scope) ?? 0); - } catch (SystemExitException ex) { - object otherCode; - res = ex.GetExitCode(out otherCode); + thread.Start(); + + if (!thread.Join(testcase.Options.Timeout)) { + if(!ClrModule.IsNetCoreApp) { + thread.Abort(); + } + NUnit.Framework.TestContext.Error.WriteLine($"{testcase.Name} timed out after {testcase.Options.Timeout / 1000.0} seconds."); + res = -1; } } finally { Environment.CurrentDirectory = cwd; diff --git a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini index 6b5223d74..e83c7b6bc 100644 --- a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini @@ -4,6 +4,7 @@ Arguments="$(TEST_FILE)" IsolationLevel=PROCESS WorkingDirectory=$(TEST_FILE_DIR) Redirect=false +Timeout=600000 ; 10 minute timeout [IronPython.test_cliclass] RunCondition='$(OS)' != 'Unix' diff --git a/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini b/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini index de017a1eb..22fb09c0f 100644 --- a/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini @@ -2,6 +2,7 @@ Ignore=false WorkingDirectory=$(TEST_FILE_DIR) Redirect=false +Timeout=600000 ; 10 minute timeout [StandardCPython.test_builtin] Ignore=true diff --git a/Src/IronPythonTest/Util/TestManifest.cs b/Src/IronPythonTest/Util/TestManifest.cs index e692d2c66..e7ccd2d16 100644 --- a/Src/IronPythonTest/Util/TestManifest.cs +++ b/Src/IronPythonTest/Util/TestManifest.cs @@ -112,7 +112,7 @@ public bool Redirect { public int Timeout { get { - return this.manifest.GetInt(this.testName, "Timeout", -1); + return this.manifest.GetInt(this.testName, "Timeout", System.Threading.Timeout.Infinite); } } From 85c0c2eca9a589d9f5c6c05dbd4c8943e78d75bd Mon Sep 17 00:00:00 2001 From: Alex Earl Date: Mon, 27 Aug 2018 11:16:06 -0700 Subject: [PATCH 2/2] Change to using Tasks --- Src/IronPythonTest/Cases/CPythonCases.cs | 27 ++++++++++++++++++--- Src/IronPythonTest/Cases/CaseExecuter.cs | 21 +++++----------- Src/IronPythonTest/Cases/IronPythonCases.cs | 9 ++++++- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/Src/IronPythonTest/Cases/CPythonCases.cs b/Src/IronPythonTest/Cases/CPythonCases.cs index d99e77721..eb08df983 100644 --- a/Src/IronPythonTest/Cases/CPythonCases.cs +++ b/Src/IronPythonTest/Cases/CPythonCases.cs @@ -27,7 +27,14 @@ public int StandardCPythonTests(TestInfo testcase) { TestContext.Progress.WriteLine(testcase.Name); // should be printed immediately return executor.RunTest(testcase); } catch (Exception e) { - Assert.Fail(executor.FormatException(e)); + if(e is AggregateException ae) { + ae.Handle((x) => { + Assert.Fail(executor.FormatException(x)); + return true; + }); + } else { + Assert.Fail(executor.FormatException(e)); + } return -1; } } @@ -48,7 +55,14 @@ public int AllCPythonTests(TestInfo testcase) { TestContext.Progress.WriteLine(testcase.Name); // should be printed immediately return executor.RunTest(testcase); } catch (Exception e) { - Assert.Fail(executor.FormatException(e)); + if(e is AggregateException ae) { + ae.Handle((x) => { + Assert.Fail(executor.FormatException(x)); + return true; + }); + } else { + Assert.Fail(executor.FormatException(e)); + } return -1; } } @@ -69,7 +83,14 @@ public int CTypesCPythonTests(TestInfo testcase) { TestContext.Progress.WriteLine(testcase.Name); // should be printed immediately return executor.RunTest(testcase); } catch (Exception e) { - Assert.Fail(executor.FormatException(e)); + if(e is AggregateException ae) { + ae.Handle((x) => { + Assert.Fail(executor.FormatException(x)); + return true; + }); + } else { + Assert.Fail(executor.FormatException(e)); + } return -1; } } diff --git a/Src/IronPythonTest/Cases/CaseExecuter.cs b/Src/IronPythonTest/Cases/CaseExecuter.cs index 86fb93b35..7f702c682 100644 --- a/Src/IronPythonTest/Cases/CaseExecuter.cs +++ b/Src/IronPythonTest/Cases/CaseExecuter.cs @@ -258,24 +258,15 @@ private int GetResult(TestInfo testcase, ScriptEngine engine, ScriptSource sourc engine.GetSysModule().SetVariable("argv", List.FromArrayNoCopy(new object[] { source.Path })); var compiledCode = source.Compile(new IronPython.Compiler.PythonCompilerOptions() { ModuleName = "__main__" }); - var thread = new Thread(() => { + var task = Task.Run(() => { try { - res = engine.Operations.ConvertTo(compiledCode.Execute(scope) ?? 0); + return engine.Operations.ConvertTo(compiledCode.Execute(scope) ?? 0); } catch (SystemExitException ex) { - res = ex.GetExitCode(out object otherCode); - } catch (ThreadAbortException) { - Thread.ResetAbort(); - } - }) { - IsBackground = true - }; - - thread.Start(); - - if (!thread.Join(testcase.Options.Timeout)) { - if(!ClrModule.IsNetCoreApp) { - thread.Abort(); + return ex.GetExitCode(out object otherCode); } + }); + + if (!task.Wait(testcase.Options.Timeout)) { NUnit.Framework.TestContext.Error.WriteLine($"{testcase.Name} timed out after {testcase.Options.Timeout / 1000.0} seconds."); res = -1; } diff --git a/Src/IronPythonTest/Cases/IronPythonCases.cs b/Src/IronPythonTest/Cases/IronPythonCases.cs index 0cb823d23..7d5bbf3b7 100644 --- a/Src/IronPythonTest/Cases/IronPythonCases.cs +++ b/Src/IronPythonTest/Cases/IronPythonCases.cs @@ -21,7 +21,14 @@ public int IronPythonTests(TestInfo testcase) { TestContext.Progress.WriteLine(testcase.Name); return this.executor.RunTest(testcase); } catch (Exception e) { - Assert.Fail(this.executor.FormatException(e)); + if(e is AggregateException ae) { + ae.Handle((x) => { + Assert.Fail(executor.FormatException(x)); + return true; + }); + } else { + Assert.Fail(this.executor.FormatException(e)); + } return -1; } }