diff --git a/Src/IronPython.Modules/binascii.cs b/Src/IronPython.Modules/binascii.cs index d7dcdbff8..82063a0aa 100644 --- a/Src/IronPython.Modules/binascii.cs +++ b/Src/IronPython.Modules/binascii.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -11,9 +12,8 @@ using IronPython.Runtime.Exceptions; using IronPython.Runtime.Operations; using IronPython.Runtime.Types; -using Microsoft.Scripting; + using Microsoft.Scripting.Runtime; -using Microsoft.Scripting.Utils; [assembly: PythonModule("binascii", typeof(IronPython.Modules.PythonBinaryAscii))] namespace IronPython.Modules { @@ -295,8 +295,48 @@ public static object rlecode_hqx(object data) { public static object b2a_hqx(object data) { throw new NotImplementedException(); } - public static object crc_hqx(object data, object crc) { - throw new NotImplementedException(); + + private static ushort[] crctab_hqx = new ushort[] { + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, + 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, + 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, + 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, + 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, + 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, + 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, + 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, + 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, + 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, + 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, + 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, + 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, + 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, + 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, + 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, + 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, + 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, + 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, + 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, + 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, + 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, + 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, + 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, + 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, + 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, + 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, + 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, + 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, + }; + + public static object crc_hqx([BytesConversion]IList data, int crc) { + crc = crc & 0xffff; + foreach (var b in data) { + crc = ((crc << 8) & 0xff00) ^ crctab_hqx[(crc >> 8) ^ b]; + } + return crc; } [Documentation("crc32(string[, value]) -> string\n\nComputes a CRC (Cyclic Redundancy Check) checksum of string.")] diff --git a/Src/IronPython.Modules/re.cs b/Src/IronPython.Modules/re.cs index 5c20da9fb..3642b563b 100644 --- a/Src/IronPython.Modules/re.cs +++ b/Src/IronPython.Modules/re.cs @@ -211,7 +211,7 @@ public class RE_Pattern : IWeakReferenceable { internal ParsedRegex _pre; internal RE_Pattern(CodeContext/*!*/ context, object pattern, int flags=0, bool compiled=false) { - _pre = PreParseRegex(context, ValidatePatternAsString(pattern)); + _pre = PreParseRegex(context, ValidatePatternAsString(pattern), (flags & VERBOSE) != 0); try { flags |= OptionToFlags(_pre.Options); RegexOptions opts = FlagsToOption(flags); @@ -876,13 +876,15 @@ public ParsedRegex(string pattern) { public RegexOptions Options = RegexOptions.CultureInvariant; } - private static readonly char[] _preParsedChars = new[] { '(', '{', '[', ']' }; + private static readonly char[] _endOfLineChars = new[] { '\r', '\n' }; + private static readonly char[] _preParsedChars = new[] { '(', '{', '[', ']', '#' }; private const string _mangledNamedGroup = "___PyRegexNameMangled"; + /// /// Preparses a regular expression text returning a ParsedRegex class /// that can be used for further regular expressions. /// - private static ParsedRegex PreParseRegex(CodeContext/*!*/ context, string pattern) { + private static ParsedRegex PreParseRegex(CodeContext/*!*/ context, string pattern, bool verbose) { ParsedRegex res = new ParsedRegex(pattern); //string newPattern; @@ -890,11 +892,20 @@ private static ParsedRegex PreParseRegex(CodeContext/*!*/ context, string patter int curGroup = 0; bool isCharList = false; bool containsNamedGroup = false; + bool inComment = false; int groupCount = 0; var namedGroups = new Dictionary(); for (; ; ) { + if (verbose && inComment) { + // read to end of line + inComment = false; + var idx = pattern.IndexOfAny(_endOfLineChars, cur); + if (idx < 0) break; + cur = idx; + } + nameIndex = pattern.IndexOfAny(_preParsedChars, cur); if (nameIndex > 0 && pattern[nameIndex - 1] == '\\') { int curIndex = nameIndex - 2; @@ -906,7 +917,7 @@ private static ParsedRegex PreParseRegex(CodeContext/*!*/ context, string patter // odd number of back slashes, this is an optional // paren that we should ignore. if ((backslashCount & 0x01) != 0) { - cur++; + cur = ++nameIndex; continue; } } @@ -930,6 +941,12 @@ private static ParsedRegex PreParseRegex(CodeContext/*!*/ context, string patter nameIndex++; isCharList = false; break; + case '#': + if (verbose && !isCharList) { + inComment = true; + } + nameIndex++; + break; case '(': // make sure we're not dealing with [(] if (!isCharList) { diff --git a/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini b/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini index 211ad81c1..7a8147f09 100644 --- a/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/AllCPythonCasesManifest.ini @@ -3,6 +3,7 @@ Ignore=false Arguments="$(TEST_FILE)" WorkingDirectory=$(TEST_FILE_DIR) Redirect=false +Timeout=600000 ; 10 minute timeout [AllCPython.test___all__] Ignore=true @@ -64,9 +65,6 @@ Ignore=true [AllCPython.test_binhex] Ignore=true -[AllCPython.test_bisect] -FullFrames=true - [AllCPython.test_bsddb3] Ignore=true @@ -190,7 +188,6 @@ Ignore=true Ignore=true [AllCPython.test_cookie] -FullFrames=true Ignore=true [AllCPython.test_copy] @@ -221,18 +218,15 @@ Ignore=true Ignore=true [AllCPython.test_decimal] -FullFrames=true Ignore=true [AllCPython.test_defaultdict] Ignore=true [AllCPython.test_deque] -FullFrames=true Ignore=true [AllCPython.test_descrtut] -FullFrames=true Ignore=true [AllCPython.test_descr] @@ -259,11 +253,7 @@ Ignore=true [AllCPython.test_dl] Ignore=true -[AllCPython.test_doctest2] -FullFrames=true - [AllCPython.test_doctest] -FullFrames=true Ignore=true [AllCPython.test_docxmlrpc] @@ -292,7 +282,6 @@ Ignore=true Ignore=true [AllCPython.test_extcall] -FullFrames=true Ignore=true [AllCPython.test_fcntl] @@ -369,9 +358,6 @@ Ignore=true [AllCPython.test_getargs2] Ignore=true -[AllCPython.test_getopt] -FullFrames=truet - [AllCPython.test_gettext] IsolationLevel=ENGINE @@ -431,7 +417,7 @@ IsolationLevel=ENGINE [AllCPython.test_import_magic] Ignore=true -Reason=IronPython doesn't implement get_magic +Reason=IronPython doesnt implement get_magic [AllCPython.test_index] Ignore=true @@ -449,6 +435,10 @@ IsolationLevel=ENGINE [AllCPython.test_ioctl] Ignore=true +[AllCPython.test_isinstance] +IsolationLevel=ENGINE +MaxRecursion=100 + [AllCPython.test_iterlen] Ignore=true @@ -594,11 +584,9 @@ Ignore=true Ignore=true [AllCPython.test_pickletools] -FullFrames=true Ignore=true [AllCPython.test_pickle] -FullFrames=true Ignore=true [AllCPython.test_pipes] @@ -701,9 +689,6 @@ Ignore=true [AllCPython.test_select] Ignore=true -[AllCPython.test_sets] -FullFrames=true - [AllCPython.test_set] Ignore=true @@ -808,7 +793,6 @@ Ignore=true Ignore=true [AllCPython.test_syntax] -FullFrames=true Ignore=true [AllCPython.test_sys] @@ -867,9 +851,6 @@ Ignore=true [AllCPython.test_tk] Ignore=true -[AllCPython.test_tokenize] -FullFrames=true - [AllCPython.test_tools] Ignore=true @@ -919,9 +900,6 @@ Ignore=true Ignore=true IsolationLevel=ENGINE -[AllCPython.test_unpack] -FullFrames=true - [AllCPython.test_urllib] Ignore=true Reason=Intermittent StackOverFlowException - https://github.com/IronLanguages/ironpython2/issues/40 @@ -930,7 +908,6 @@ Reason=Intermittent StackOverFlowException - https://github.com/IronLanguages/ir Ignore=true [AllCPython.test_urllib2net] -FullFrames=true Ignore=true [AllCPython.test_urllib2_localnet] diff --git a/Src/IronPythonTest/Cases/CPythonCases.cs b/Src/IronPythonTest/Cases/CPythonCases.cs index d99e77721..50072d40a 100644 --- a/Src/IronPythonTest/Cases/CPythonCases.cs +++ b/Src/IronPythonTest/Cases/CPythonCases.cs @@ -1,4 +1,8 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections; using System.Collections.Generic; using System.IO; @@ -13,69 +17,30 @@ namespace IronPythonTest.Cases { [TestFixture(Category = "StandardCPython")] - class StandardCPythonCases { - private CaseExecuter executor; - - [OneTimeSetUp] - public void FixtureSetUp() { - executor = new CaseExecuter(); - } - + public class StandardCPythonCases : CommonCases { [Test, TestCaseSource(typeof(StandardCPythonCaseGenerator))] - public int StandardCPythonTests(TestInfo testcase) { - try { - TestContext.Progress.WriteLine(testcase.Name); // should be printed immediately - return executor.RunTest(testcase); - } catch (Exception e) { - Assert.Fail(executor.FormatException(e)); - return -1; - } + public override int Test(TestInfo testcase) { + return TestImpl(testcase); } } [TestFixture(Category = "AllCPython")] - class AllCPythonCases { - private CaseExecuter executor; - - [OneTimeSetUp] - public void FixtureSetUp() { - executor = new CaseExecuter(); - } - + public class AllCPythonCases : CommonCases { [Test, TestCaseSource(typeof(AllCPythonCaseGenerator))] - public int AllCPythonTests(TestInfo testcase) { - try { - TestContext.Progress.WriteLine(testcase.Name); // should be printed immediately - return executor.RunTest(testcase); - } catch (Exception e) { - Assert.Fail(executor.FormatException(e)); - return -1; - } + public override int Test(TestInfo testcase) { + return TestImpl(testcase); } } [TestFixture(Category = "CTypesCPython")] - class CTypesCPythonCases { - private CaseExecuter executor; - - [OneTimeSetUp] - public void FixtureSetUp() { - executor = new CaseExecuter(); - } - + public class CTypesCPythonCases : CommonCases { [Test, TestCaseSource(typeof(CTypesCPythonCaseGenerator))] - public int CTypesCPythonTests(TestInfo testcase) { - try { - TestContext.Progress.WriteLine(testcase.Name); // should be printed immediately - return executor.RunTest(testcase); - } catch (Exception e) { - Assert.Fail(executor.FormatException(e)); - return -1; - } + public override int Test(TestInfo testcase) { + return TestImpl(testcase); } } - class StandardCPythonCaseGenerator : CommonCaseGenerator { + internal class StandardCPythonCaseGenerator : CommonCaseGenerator { internal static readonly HashSet STDTESTS = new HashSet { "test_grammar", "test_opcodes", @@ -96,7 +61,7 @@ protected override IEnumerable GetTests() { } } - class AllCPythonCaseGenerator : CommonCaseGenerator { + internal class AllCPythonCaseGenerator : CommonCaseGenerator { protected override IEnumerable GetTests() { var testDir = Path.Combine("Src", "StdLib", "Lib", "test"); var fullPath = Path.Combine(CaseExecuter.FindRoot(), testDir); @@ -107,7 +72,7 @@ protected override IEnumerable GetTests() { } } - class CTypesCPythonCaseGenerator : CommonCaseGenerator { + internal class CTypesCPythonCaseGenerator : CommonCaseGenerator { protected override IEnumerable GetTests() { var testDir = Path.Combine("Src", "StdLib", "Lib", "ctypes", "test"); var fullPath = Path.Combine(CaseExecuter.FindRoot(), testDir); diff --git a/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini b/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini index 7c443f7fe..d271689bb 100644 --- a/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/CTypesCPythonCasesManifest.ini @@ -4,6 +4,7 @@ Arguments="$(TEST_FILE)" WorkingDirectory=$(TEST_FILE_DIR) Redirect=false RunCondition=NOT $(IS_MACOS) +Timeout=600000 ; 10 minute timeout [CTypesCPython.test_byteswap] RetryCount=2 diff --git a/Src/IronPythonTest/Cases/CaseExecuter.cs b/Src/IronPythonTest/Cases/CaseExecuter.cs index cfb3c0e4a..9edfceefd 100644 --- a/Src/IronPythonTest/Cases/CaseExecuter.cs +++ b/Src/IronPythonTest/Cases/CaseExecuter.cs @@ -10,6 +10,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Microsoft.Scripting; using Microsoft.Scripting.Hosting; @@ -20,7 +21,7 @@ using IronPythonTest.Util; namespace IronPythonTest.Cases { - class CaseExecuter { + internal class CaseExecuter { private static string Executable { get { var folder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); @@ -85,9 +86,9 @@ internal static string FindRoot() { private static void AddSearchPaths(ScriptEngine engine) { var paths = new List(engine.GetSearchPaths()); - if(!paths.Any(x => x.ToLower().Contains("stdlib"))) { + if (!paths.Any(x => x.ToLower().Contains("stdlib"))) { var root = FindRoot(); - if(!string.IsNullOrEmpty(root)) { + if (!string.IsNullOrEmpty(root)) { paths.Insert(0, Path.Combine(root, "Src", "StdLib", "Lib")); } } @@ -95,12 +96,7 @@ private static void AddSearchPaths(ScriptEngine engine) { } public CaseExecuter() { - this.defaultEngine = Python.CreateEngine(new Dictionary { - {"Debug", false}, - {"Frames", true}, - {"FullFrames", false}, - {"RecursionLimit", 100} - }); + this.defaultEngine = Python.CreateEngine(); this.defaultEngine.SetHostVariables( Path.GetDirectoryName(Executable), @@ -111,32 +107,19 @@ public CaseExecuter() { public int RunTest(TestInfo testcase) { int retryCount = testcase.Options.RetryCount; - if(retryCount > 0) { - int res = -1; - for(int i = 0; i < retryCount; i++) { - try { - res = RunTestImpl(testcase); - } catch(Exception ex) { - res = -1; - if(i == (retryCount - 1)) { - throw ex; - } - } - - if(res != 0) { - NUnit.Framework.TestContext.Progress.WriteLine($"Test {testcase.Name} failed, retrying again. Retry #{i+1}"); - } else { - break; - } - } - return res; - } - + for (int i = 0; i < retryCount; i++) { + try { + var res = RunTestImpl(testcase); + if (res == 0) return res; + NUnit.Framework.TestContext.Progress.WriteLine($"Test {testcase.Name} failed, retrying again. Retry #{i + 1}"); + } catch { } + } + return RunTestImpl(testcase); } - + private int RunTestImpl(TestInfo testcase) { - switch(testcase.Options.IsolationLevel) { + switch (testcase.Options.IsolationLevel) { case TestIsolationLevel.DEFAULT: return GetScopeTest(testcase); @@ -163,9 +146,9 @@ private static string GetIronPythonPath() { return string.Empty; } - private string ReplaceVariables(string input, IDictionary replacements) { + private string ReplaceVariables(string input, IDictionary replacements) { Regex variableRegex = new Regex(@"\$\(([^}]+)\)", RegexOptions.Compiled); - + var result = input; var match = variableRegex.Match(input); while (match.Success) { @@ -184,7 +167,7 @@ private int GetEngineTest(TestInfo testcase) { var source = engine.CreateScriptSourceFromString( testcase.Text, testcase.Path, SourceCodeKind.File); - return GetResult(engine, source, testcase.Path, testcase.Options.WorkingDirectory); + return GetResult(testcase, engine, source, testcase.Path, testcase.Options.WorkingDirectory); } private int GetProcessTest(TestInfo testcase) { @@ -199,9 +182,23 @@ private int GetProcessTest(TestInfo testcase) { { "TEST_FILE_DIR", Path.GetDirectoryName(testcase.Path) } }; + // add the arguments - in the normal case no arguments should be added + var arguments = new List(); + if (testcase.Options.Debug) + arguments.Add("-X:Debug"); + if (!testcase.Options.Frames) + arguments.Add("-X:NoFrames"); + if (testcase.Options.FullFrames) + arguments.Add("-X:FullFrames"); + if (testcase.Options.MaxRecursion != int.MaxValue) + arguments.Add($"-X:MaxRecursion {testcase.Options.MaxRecursion}"); + if (testcase.Options.Tracing) + arguments.Add("-X:Tracing"); + arguments.Add(ReplaceVariables(testcase.Options.Arguments, argReplacements)); + using (Process proc = new Process()) { proc.StartInfo.FileName = Executable; - proc.StartInfo.Arguments = ReplaceVariables(testcase.Options.Arguments, argReplacements); + proc.StartInfo.Arguments = string.Join(" ", arguments); if (!string.IsNullOrEmpty(IRONPYTHONPATH)) { proc.StartInfo.EnvironmentVariables["IRONPYTHONPATH"] = IRONPYTHONPATH; @@ -216,13 +213,13 @@ private int GetProcessTest(TestInfo testcase) { proc.Start(); if (testcase.Options.Redirect) { - AsyncStreamReader(proc.StandardOutput, data => Console.Write(data)); - AsyncStreamReader(proc.StandardError, data => Console.Error.Write(data)); + AsyncStreamReader(proc.StandardOutput, data => NUnit.Framework.TestContext.Out.Write(data)); + AsyncStreamReader(proc.StandardError, data => NUnit.Framework.TestContext.Error.Write(data)); } if (!proc.WaitForExit(testcase.Options.Timeout)) { proc.Kill(); - Console.Error.Write($"Timed out after {testcase.Options.Timeout / 1000.0} seconds."); + NUnit.Framework.TestContext.Error.WriteLine($"{testcase.Name} timed out after {testcase.Options.Timeout / 1000.0} seconds."); } exitCode = proc.ExitCode; } @@ -249,21 +246,28 @@ void ReadCallback(IAsyncResult asyncResult) { } private int GetScopeTest(TestInfo testcase) { + if (testcase.Options.Debug + || !testcase.Options.Frames + || testcase.Options.FullFrames + || testcase.Options.MaxRecursion != int.MaxValue + || testcase.Options.Tracing) { + throw new Exception("Options have no effect with IsolationLevel=DEFAULT, use ENGINE or PROCESS instead."); + } + var source = this.defaultEngine.CreateScriptSourceFromString( testcase.Text, testcase.Path, SourceCodeKind.File); - return GetResult(this.defaultEngine, source, testcase.Path, testcase.Options.WorkingDirectory); + return GetResult(testcase, this.defaultEngine, source, testcase.Path, testcase.Options.WorkingDirectory); } - private int GetResult(ScriptEngine engine, ScriptSource source, string testPath, string workingDir) { - int res = 0; + private int GetResult(TestInfo testcase, ScriptEngine engine, ScriptSource source, string testPath, string workingDir) { var path = Environment.GetEnvironmentVariable("IRONPYTHONPATH"); - if(string.IsNullOrEmpty(path)) { + if (string.IsNullOrEmpty(path)) { Environment.SetEnvironmentVariable("IRONPYTHONPATH", IRONPYTHONPATH); } var cwd = Environment.CurrentDirectory; - if(!string.IsNullOrWhiteSpace(workingDir)) { + if (!string.IsNullOrWhiteSpace(workingDir)) { var replacements = new Dictionary() { { "ROOT", FindRoot() }, { "TEST_FILE_DIR", Path.GetDirectoryName(testPath) } @@ -276,16 +280,21 @@ private int GetResult(ScriptEngine engine, ScriptSource source, string testPath, engine.GetSysModule().SetVariable("argv", List.FromArrayNoCopy(new object[] { source.Path })); var compiledCode = source.Compile(new IronPython.Compiler.PythonCompilerOptions() { ModuleName = "__main__" }); - try { - res = engine.Operations.ConvertTo(compiledCode.Execute(scope) ?? 0); - } catch (SystemExitException ex) { - object otherCode; - res = ex.GetExitCode(out otherCode); + var task = Task.Run(() => { + try { + return engine.Operations.ConvertTo(compiledCode.Execute(scope) ?? 0); + } catch (SystemExitException ex) { + return ex.GetExitCode(out _); + } + }); + if (!task.Wait(testcase.Options.Timeout)) { + NUnit.Framework.TestContext.Error.WriteLine($"{testcase.Name} timed out after {testcase.Options.Timeout / 1000.0} seconds."); + return -1; } + return task.Result; } finally { Environment.CurrentDirectory = cwd; } - return res; } } } diff --git a/Src/IronPythonTest/Cases/CaseGenerator.cs b/Src/IronPythonTest/Cases/CaseGenerator.cs index 756773ad0..e0645fd12 100644 --- a/Src/IronPythonTest/Cases/CaseGenerator.cs +++ b/Src/IronPythonTest/Cases/CaseGenerator.cs @@ -1,17 +1,22 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections; using System.Collections.Generic; using System.Data; using System.IO; -using System.Linq; -using System.Text; +using System.Runtime.InteropServices; + +using IronPython; using IronPythonTest.Util; + using NUnit.Framework; -using NUnit.Framework.Api; namespace IronPythonTest.Cases { - public class TestInfo { - public TestInfo(string path, string baseName, string rootDir, TestManifest testManifest) { + public class TestInfo { + public TestInfo(string path, string baseName, string rootDir, TestManifest testManifest) { this.Path = path; this.Text = LoadTest(path); this.Name = GetName(path, baseName, rootDir); @@ -30,7 +35,7 @@ private static string LoadTest(string path) { private static string GetName(string path, string baseName, string rootDir) { var root = CaseExecuter.FindRoot(); var dir = System.IO.Path.GetDirectoryName(path).Replace(root, string.Empty).Replace(rootDir, string.Empty).Replace('\\', '.').Replace('/', '.').TrimStart('.'); - if(string.IsNullOrWhiteSpace(dir)) { + if (string.IsNullOrWhiteSpace(dir)) { return $"{baseName}.{System.IO.Path.GetFileNameWithoutExtension(path)}"; } return $"{baseName}.{dir}.{System.IO.Path.GetFileNameWithoutExtension(path)}"; @@ -39,9 +44,9 @@ private static string GetName(string path, string baseName, string rootDir) { public override string ToString() { return this.Name; } - } + } - abstract class CommonCaseGenerator : IEnumerable { + internal abstract class CommonCaseGenerator : IEnumerable { protected readonly TestManifest manifest = new TestManifest(typeof(TCases)); protected static readonly string category = ((TestFixtureAttribute)typeof(TCases).GetCustomAttributes(typeof(TestFixtureAttribute), false)[0]).Category; @@ -49,10 +54,10 @@ public IEnumerator GetEnumerator() { foreach (var testcase in GetTests()) { var name = testcase.Name; var framework = TestContext.Parameters["FRAMEWORK"]; - if(!string.IsNullOrWhiteSpace(framework)) { + if (!string.IsNullOrWhiteSpace(framework)) { name = $"{framework}.{testcase.Name}"; } - + var result = new TestCaseData(testcase) .SetCategory(category) .SetName(name) @@ -66,7 +71,7 @@ public IEnumerator GetEnumerator() { } } - if(!ConditionMatched(testcase.Options.RunCondition) && string.IsNullOrWhiteSpace(TestContext.Parameters["RUN_IGNORED"])) { + if (!ConditionMatched(testcase.Options.RunCondition) && string.IsNullOrWhiteSpace(TestContext.Parameters["RUN_IGNORED"])) { if (!string.IsNullOrWhiteSpace(testcase.Options.Reason)) { result.Ignore($"condition ({testcase.Options.RunCondition}) - {testcase.Options.Reason}"); } else { @@ -82,10 +87,10 @@ public IEnumerator GetEnumerator() { protected bool ConditionMatched(string condition) { bool result = true; - if(!string.IsNullOrEmpty(condition)) { + if (!string.IsNullOrEmpty(condition)) { try { result = EvaluateExpression(condition); - } catch(Exception ex) { + } catch (Exception ex) { Console.WriteLine($"Error evaluating test condition '{condition}', will run the test anyway: {ex.Message}"); result = true; } @@ -99,10 +104,15 @@ private bool EvaluateExpression(string expression) { string filter = expression; var replacements = new Dictionary() { // variables - { "$(OS)", Environment.OSVersion.Platform.ToString() }, { "$(IS_NETCOREAPP)", IronPython.Runtime.ClrModule.IsNetCoreApp.ToString() }, { "$(IS_MONO)", IronPython.Runtime.ClrModule.IsMono.ToString() }, { "$(IS_DEBUG)", IronPython.Runtime.ClrModule.IsDebug.ToString() }, + { "$(IS_POSIX)", (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.IsOSPlatform(OSPlatform.Linux)).ToString() }, + { "$(IS_OSX)", RuntimeInformation.IsOSPlatform(OSPlatform.OSX).ToString() }, + { "$(IS_LINUX)", RuntimeInformation.IsOSPlatform(OSPlatform.Linux).ToString() }, + { "$(IS_WINDOWS)", RuntimeInformation.IsOSPlatform(OSPlatform.Windows).ToString() }, + + { "$(OS)", Environment.OSVersion.Platform.ToString() }, { "$(IS_MACOS)", IronPython.Runtime.ClrModule.IsMacOS.ToString() }, // operators @@ -127,7 +137,7 @@ private bool EvaluateExpression(string expression) { if (ex.Message.StartsWith("The expression contains undefined function call", StringComparison.Ordinal)) throw new Exception("A variable used in the filter expression is not defined"); throw new Exception($"Invalid filter: {ex.Message}"); - } catch(SyntaxErrorException ex) { + } catch (SyntaxErrorException ex) { throw new Exception($"Invalid filter: {ex.Message}"); } diff --git a/Src/IronPythonTest/Cases/CommonCases.cs b/Src/IronPythonTest/Cases/CommonCases.cs new file mode 100644 index 000000000..e8636016c --- /dev/null +++ b/Src/IronPythonTest/Cases/CommonCases.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; +using NUnit.Framework; + +namespace IronPythonTest.Cases { + public abstract class CommonCases { + private CaseExecuter executor; + + [OneTimeSetUp] + public void FixtureSetUp() { + this.executor = new CaseExecuter(); + } + + public abstract int Test(TestInfo testcase); + + protected int TestImpl(TestInfo testcase) { + try { + TestContext.Progress.WriteLine(testcase.Name); + return executor.RunTest(testcase); + } catch (Exception e) { + if (e is AggregateException ae) { + ae.Handle((x) => { + Assert.Fail(executor.FormatException(x)); + return true; + }); + } else { + Assert.Fail(this.executor.FormatException(e)); + } + return -1; + } + } + } +} diff --git a/Src/IronPythonTest/Cases/IronPythonCases.cs b/Src/IronPythonTest/Cases/IronPythonCases.cs index 149db2437..767cd3eef 100644 --- a/Src/IronPythonTest/Cases/IronPythonCases.cs +++ b/Src/IronPythonTest/Cases/IronPythonCases.cs @@ -1,36 +1,26 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections.Generic; using System.IO; using System.Linq; using NUnit.Framework; - namespace IronPythonTest.Cases { - [TestFixture(Category="IronPython")] - public class IronPythonCases { - private CaseExecuter executor; - - [OneTimeSetUp] - public void FixtureSetUp() { - this.executor = new CaseExecuter(); - } - + [TestFixture(Category = "IronPython")] + public class IronPythonCases : CommonCases { [Test, TestCaseSource(typeof(IronPythonCaseGenerator))] - public int IronPythonTests(TestInfo testcase) { - try { - TestContext.Progress.WriteLine(testcase.Name); - return this.executor.RunTest(testcase); - } catch (Exception e) { - Assert.Fail(this.executor.FormatException(e)); - return -1; - } + public override int Test(TestInfo testcase) { + return TestImpl(testcase); } } - class IronPythonCaseGenerator : CommonCaseGenerator { + internal class IronPythonCaseGenerator : CommonCaseGenerator { protected override IEnumerable GetTests() { var root = CaseExecuter.FindRoot(); - + return GetFilenames() .Select(file => new TestInfo(Path.GetFullPath(file), category, "Tests", this.manifest)) .OrderBy(testcase => testcase.Name); diff --git a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini index 8a005fdde..ece7acb4b 100644 --- a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini @@ -4,6 +4,7 @@ Arguments="$(TEST_FILE)" IsolationLevel=PROCESS WorkingDirectory=$(TEST_FILE_DIR) Redirect=false +Timeout=600000 ; 10 minute timeout [IronPython.test_python25] Ignore=true @@ -84,7 +85,7 @@ RetryCount=2 RunCondition=NOT $(IS_MACOS) [IronPython.modules.system_related.test_sys_getframe] -Arguments=-X:FullFrames "$(TEST_FILE)" +FullFrames=true [IronPython.test_traceback] Ignore=true diff --git a/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini b/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini index bd1ac0b79..f6079b0e8 100644 --- a/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/StandardCPythonCasesManifest.ini @@ -2,6 +2,7 @@ Ignore=false WorkingDirectory=$(TEST_FILE_DIR) Redirect=false +Timeout=600000 ; 10 minute timeout [StandardCPython.test___all__] Ignore=true @@ -16,9 +17,6 @@ Ignore=true [StandardCPython.test_doctest] Ignore=true -[StandardCPython.test_doctest2] -FullFrames=true - [StandardCPython.test_exceptions] Ignore=true diff --git a/Src/IronPythonTest/Util/IniParser.cs b/Src/IronPythonTest/Util/IniParser.cs index 5761b476a..eecede512 100644 --- a/Src/IronPythonTest/Util/IniParser.cs +++ b/Src/IronPythonTest/Util/IniParser.cs @@ -1,4 +1,8 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections.Generic; using System.IO; @@ -7,7 +11,7 @@ namespace IronPythonTest.Util { using OptionStore = Dictionary>; public class IniParser { - OptionStore options; + private OptionStore options; public IniParser(Stream source) { this.options = Parse(new StreamReader(source).ReadLines()); @@ -20,19 +24,20 @@ public string GetValue(string sectionName, string key) { public string GetValue(string sectionName, string key, string @default) { sectionName = string.IsNullOrEmpty(sectionName) ? "DEFAULT" : sectionName; - Section section; - if (!this.options.TryGetValue(sectionName, out section)) { - section = this.options["DEFAULT"]; - } - string value; - if (!section.TryGetValue(key, out value)) { - if (!this.options["DEFAULT"].TryGetValue(key, out value)) { - return @default; + while ((sectionName = GetParentSection(sectionName, out Section section)) != null) { + if (section.TryGetValue(key, out value)) { + return value; } } - return value; + return options["DEFAULT"].TryGetValue(key, out value) ? value : @default; + } + + private string GetParentSection(string sectionName, out Section section) { + var idx = sectionName.LastIndexOf('.'); + var newSectionName = idx == -1 ? null : sectionName.Substring(0, idx); + return options.TryGetValue(sectionName, out section) || newSectionName == null ? newSectionName : GetParentSection(newSectionName, out section); } public bool GetBool(string sectionName, string key) { @@ -87,7 +92,7 @@ private static OptionStore Parse(IEnumerable lines) { } } - static class TextReaderExtensions { + internal static class TextReaderExtensions { public static IEnumerable ReadLines(this TextReader tr) { string line; while ((line = tr.ReadLine()) != null) { @@ -96,9 +101,9 @@ public static IEnumerable ReadLines(this TextReader tr) { } } - static class StringExtensions { - static HashSet Truthy = new HashSet { "1", "t", "true", "y", "yes" }; - static HashSet Falsey = new HashSet { "0", "f", "false", "n", "no" }; + internal static class StringExtensions { + private static HashSet Truthy = new HashSet { "1", "t", "true", "y", "yes" }; + private static HashSet Falsey = new HashSet { "0", "f", "false", "n", "no" }; public static bool AsBool(this string s) { if (s == null) { diff --git a/Src/IronPythonTest/Util/TestManifest.cs b/Src/IronPythonTest/Util/TestManifest.cs index 46deffaf0..a28a2b4c6 100644 --- a/Src/IronPythonTest/Util/TestManifest.cs +++ b/Src/IronPythonTest/Util/TestManifest.cs @@ -1,4 +1,8 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -6,7 +10,7 @@ namespace IronPythonTest.Util { public class TestManifest { - IniParser manifest; + private IniParser manifest; public TestManifest(Type parent) { var file = parent.Assembly.GetManifestResourceStream($"IronPythonTest.Cases.{parent.Name}Manifest.ini"); @@ -30,8 +34,8 @@ public enum TestIsolationLevel { } public class TestOptions { - string testName; - IniParser manifest; + private string testName; + private IniParser manifest; public TestOptions(IniParser manifest, string testName) { this.manifest = manifest; @@ -112,10 +116,10 @@ public bool Redirect { public int Timeout { get { - return this.manifest.GetInt(this.testName, "Timeout", -1); + return this.manifest.GetInt(this.testName, "Timeout", System.Threading.Timeout.Infinite); } } - + public int RetryCount { get { return this.manifest.GetInt(this.testName, "RetryCount", 0); diff --git a/Tests/modules/io_related/test_binascii.py b/Tests/modules/io_related/test_binascii.py index 80d9cfaec..0bcc893e5 100644 --- a/Tests/modules/io_related/test_binascii.py +++ b/Tests/modules/io_related/test_binascii.py @@ -36,7 +36,6 @@ def test_not_implemented(self): lambda: binascii.rledecode_hqx(None), lambda: binascii.rlecode_hqx(None), lambda: binascii.b2a_hqx(None), - lambda: binascii.crc_hqx(None, None), ] for temp_func in test_cases: self.assertRaises(NotImplementedError, temp_func) diff --git a/Tests/test_regressions.py b/Tests/test_regressions.py index a0bdfcf0d..aa4f220a5 100644 --- a/Tests/test_regressions.py +++ b/Tests/test_regressions.py @@ -1358,6 +1358,19 @@ class Test(StringIO): pass class Test(BytesIO): pass Test().seek(0) + def test_ipy3_gh546(self): + """https://github.com/IronLanguages/ironpython3/issues/546""" + import re + _SECT_TMPL = r""" + \[ # [ + (?P
[^]]+) # very permissive! + \] # ] + """ + SECTCRE = re.compile(_SECT_TMPL, re.VERBOSE) + string = "[some header]" + match = SECTCRE.match(string) + self.assertEqual(match.span(), (0, len(string))) + def test_ipy2_gh655(self): """https://github.com/IronLanguages/ironpython2/issues/655""" import pyexpat