diff --git a/.editorconfig b/.editorconfig index 97bf7d90e..d9c7ff3d7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -82,6 +82,7 @@ dotnet_diagnostic.CA1846.severity = none # CA1846: Prefer 'AsSpan' over ' dotnet_diagnostic.CA1847.severity = none # CA1847: Use char literal for a single character lookup dotnet_diagnostic.CA1852.severity = suggestion # CA1852: Seal internal types dotnet_diagnostic.CA1859.severity = suggestion # CA1859: Use concrete types when possible for improved performance +dotnet_diagnostic.CA1861.severity = suggestion # CA1861: Avoid constant arrays as arguments dotnet_diagnostic.CA2101.severity = suggestion # CA2101: Specify marshaling for P/Invoke string arguments dotnet_diagnostic.CA2201.severity = none # CA2201: Do not raise reserved exception types dotnet_diagnostic.CA2208.severity = suggestion # CA2208: Instantiate argument exceptions correctly diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index fc054e3de..c235ec2f6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ ### Prerequisites -The issue tracker is used to report bugs and request new features, NOT to ask questions. +The issue tracker is used to report bugs and request new features, **NOT** to ask questions. Questions should be posted in [Discussions](https://github.com/IronLanguages/ironpython3/discussions/categories/q-a) or to the users mailing list which can be accessed at https://ironpython.groups.io/g/users. @@ -19,10 +19,21 @@ https://ironpython.groups.io/g/users. 2. [Second Step] 3. [and so on...] -**Expected behavior:** [What you expected to happen] +**Expected behavior:** -**Actual behavior:** [What actually happened] +[What you expected to happen] -### Versions +**Actual behavior:** -You can get this information from executing `ipy -VV`. +[What actually happened] + +### Version Information + +If you are using the `ipy` console program, provide output of executing `ipy -VV`. If it is a local build, provide also the git hash of the commit used to build IronPython. In either case, provide the type of the operating system used. + +If you are using the IronPython engine embedded in a .NET application, provide the version number of the NuGet package used (or if it is a local build, the git hash of the commit used to build IronPython), and the following info: + +* .NET platform used (choice from: .NET, .NET Core, .NET Framework, Mono, Unity, Xamarin), +* Version of the .NET platform used, +* Operating system used, +* Value of `sys.version`, from imported module `sys`. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d1d39654e..0ddab2b28 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,19 +10,15 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-20.04, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] steps: - name: Install tools - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-latest' run: sudo apt-get -yq install mono-vbnc dos2unix - uses: actions/checkout@v2 with: submodules: true - - name: Setup .NET Core 2.1 - uses: actions/setup-dotnet@v1 - with: - dotnet-version: '2.1.x' - name: Setup .NET Core 3.1 uses: actions/setup-dotnet@v1 with: @@ -48,9 +44,6 @@ jobs: - name: Test (net462) run: ./make.ps1 -frameworks net462 test-all shell: pwsh - - name: Test (netcoreapp2.1) - run: ./make.ps1 -frameworks netcoreapp2.1 test-all - shell: pwsh - name: Test (netcoreapp3.1) run: ./make.ps1 -frameworks netcoreapp3.1 test-all shell: pwsh diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 49804c806..6e8567eb4 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -18,7 +18,7 @@ jobs: displayName: Linux (Ubuntu) timeoutInMinutes: 180 pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest steps: - template: Build/steps.yml parameters: diff --git a/Build/After.targets b/Build/After.targets index 10b44848a..6da59981d 100644 --- a/Build/After.targets +++ b/Build/After.targets @@ -19,8 +19,7 @@ - <_TargetFramework>$(TargetFramework) - <_TargetFramework Condition=" $(TargetFramework.EndsWith('-windows')) ">$(TargetFramework.Substring(0, $(TargetFramework.IndexOf('-windows')))) + <_TargetFramework>$(TargetFramework.Replace('-windows', '')) $(StageDir)\$(_TargetFramework) $(StageDir)\$(_TargetFramework)\DLLs @(StageItem) @@ -44,9 +43,5 @@ - - - - - + diff --git a/Build/steps.yml b/Build/steps.yml index 97d269b15..ab6945c4e 100644 --- a/Build/steps.yml +++ b/Build/steps.yml @@ -24,12 +24,6 @@ steps: Write-Host ("##vso[task.setvariable variable=PackageVersion;isSecret=false;isOutput=true;]$PackageVersion") displayName: Grab Package Version - - task: UseDotNet@2 - displayName: Install .NET Core 2.1 runtime for testing - inputs: - packageType: 'runtime' - version: '2.1.x' - - task: UseDotNet@2 displayName: Install .NET Core 3.1 runtime for testing inputs: diff --git a/IronPython.sln b/IronPython.sln index 762507dfd..d02363401 100644 --- a/IronPython.sln +++ b/IronPython.sln @@ -37,8 +37,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{17737ACB Build\net462.props = Build\net462.props Build\net6.0-windows.props = Build\net6.0-windows.props Build\net6.0.props = Build\net6.0.props - Build\netcoreapp2.1.props = Build\netcoreapp2.1.props - Build\netcoreapp3.1.props = Build\netcoreapp3.1.props Build\netstandard2.0.props = Build\netstandard2.0.props Build\steps.yml = Build\steps.yml Build\Tasks.Targets = Build\Tasks.Targets diff --git a/Package/nuget/IronPython.nuspec b/Package/nuget/IronPython.nuspec index 4708b4cb2..3cc2a5d76 100644 --- a/Package/nuget/IronPython.nuspec +++ b/Package/nuget/IronPython.nuspec @@ -25,12 +25,7 @@ This package contains the IronPython interpreter engine. - - - - - - + @@ -41,9 +36,9 @@ This package contains the IronPython interpreter engine. - - - + + + diff --git a/Package/zip/Zip.Packaging.targets b/Package/zip/Zip.Packaging.targets index b9fbbf0de..cc1f64bb8 100644 --- a/Package/zip/Zip.Packaging.targets +++ b/Package/zip/Zip.Packaging.targets @@ -4,7 +4,7 @@ - + diff --git a/README.md b/README.md index 4b35f2bf1..6d53eb37c 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,4 @@ Binaries of IronPython 3 can be downloaded from the [release page](https://githu See the [building document](Documentation/building.md). Since the main development is on Windows, bugs on other platforms may inadvertently be introduced - please report them! ## Supported Platforms -IronPython 3 targets .NET Framework 4.6.2, .NET Standard 2.0, .NET Core 3.1 and .NET 6.0. The support for .NET and .NET Core follow the lifecycle defined on [.NET and .NET Core Support Policy](https://dotnet.microsoft.com/platform/support/policy/dotnet-core). +IronPython 3 targets .NET Framework 4.6.2, .NET Standard 2.0 and .NET 6.0. The support for .NET and .NET Core follow the lifecycle defined on [.NET and .NET Core Support Policy](https://dotnet.microsoft.com/platform/support/policy/dotnet-core). diff --git a/Src/DLR b/Src/DLR index e2a679d68..872730ab3 160000 --- a/Src/DLR +++ b/Src/DLR @@ -1 +1 @@ -Subproject commit e2a679d68ab5150e97e2debcd4f35bceb0fca584 +Subproject commit 872730ab305d37f67179db8ca778d1cd22984e67 diff --git a/Src/IronPython.Modules/IronPython.Modules.csproj b/Src/IronPython.Modules/IronPython.Modules.csproj index 17538aaea..5a7b3cba0 100644 --- a/Src/IronPython.Modules/IronPython.Modules.csproj +++ b/Src/IronPython.Modules/IronPython.Modules.csproj @@ -1,7 +1,7 @@  - net462;netcoreapp3.1;netstandard2.0;net6.0 + net462;netstandard2.0;net6.0 885063680 true true @@ -33,10 +33,6 @@ - - - - diff --git a/Src/IronPython.Modules/IterTools.cs b/Src/IronPython.Modules/IterTools.cs index 582739095..e4c0db29c 100644 --- a/Src/IronPython.Modules/IterTools.cs +++ b/Src/IronPython.Modules/IterTools.cs @@ -760,11 +760,11 @@ public class zip_longest : IEnumerator { [PythonType] public class product : IterBase { - public product(params object[] iterables) { - InnerEnumerator = Yielder(ArrayUtils.ConvertAll(iterables, x => new PythonList(PythonOps.GetEnumerator(x)))); + public product(CodeContext context, params object[] iterables) { + InnerEnumerator = Yielder(ArrayUtils.ConvertAll(iterables, x => new PythonList(context, PythonOps.GetEnumerator(x)))); } - public product([ParamDictionary]IDictionary paramDict, params object[] iterables) { + public product(CodeContext context, [ParamDictionary]IDictionary paramDict, params object[] iterables) { object repeat; int iRepeat = 1; if (paramDict.TryGetValue("repeat", out repeat)) { @@ -785,7 +785,7 @@ public class product : IterBase { PythonList[] finalIterables = new PythonList[iterables.Length * iRepeat]; for (int i = 0; i < iRepeat; i++) { for (int j = 0; j < iterables.Length; j++) { - finalIterables[i * iterables.Length + j] = new PythonList(iterables[j]); + finalIterables[i * iterables.Length + j] = new PythonList(context, iterables[j]); } } InnerEnumerator = Yielder(finalIterables); @@ -840,8 +840,8 @@ public class product : IterBase { public class combinations : IterBase { private readonly PythonList _data; - public combinations(object iterable, object r) { - _data = new PythonList(iterable); + public combinations(CodeContext context, object iterable, object r) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(GetR(r, _data)); } @@ -910,8 +910,8 @@ public class combinations : IterBase { public class combinations_with_replacement : IterBase { private readonly PythonList _data; - public combinations_with_replacement(object iterable, object r) { - _data = new PythonList(iterable); + public combinations_with_replacement(CodeContext context, object iterable, object r) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(GetR(r, _data)); } @@ -979,14 +979,14 @@ public class combinations_with_replacement : IterBase { public class permutations : IterBase { private readonly PythonList _data; - public permutations(object iterable) { - _data = new PythonList(iterable); + public permutations(CodeContext context, object iterable) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(_data.Count); } - public permutations(object iterable, object r) { - _data = new PythonList(iterable); + public permutations(CodeContext context, object iterable, object r) { + _data = new PythonList(context, iterable); InnerEnumerator = Yielder(GetR(r, _data)); } @@ -1177,7 +1177,7 @@ public class starmap : IterBase { objargs[i] = args[i]; } } else { - PythonList argsList = new PythonList(PythonOps.GetEnumerator(iter.Current)); + PythonList argsList = new PythonList(context, PythonOps.GetEnumerator(iter.Current)); objargs = ArrayUtils.ToArray(argsList); } diff --git a/Src/IronPython.Modules/_datetime.cs b/Src/IronPython.Modules/_datetime.cs index e32e004e5..7d9f6a586 100644 --- a/Src/IronPython.Modules/_datetime.cs +++ b/Src/IronPython.Modules/_datetime.cs @@ -1036,7 +1036,7 @@ public datetime(DateTime dt, tzinfo? tzinfo) public double timestamp() { if (tzinfo is null) { - return PythonTime.TicksToTimestamp(_dateTime.Ticks); + return PythonTime.TicksToTimestamp(_dateTime.ToUniversalTime().Ticks); } else { return (this - new datetime(new DateTime(1970, 1, 1), timezone.utc)).total_seconds(); diff --git a/Src/IronPython.Modules/_signal.cs b/Src/IronPython.Modules/_signal.cs index a077c0be9..d7e935f6a 100644 --- a/Src/IronPython.Modules/_signal.cs +++ b/Src/IronPython.Modules/_signal.cs @@ -118,13 +118,13 @@ public static partial class PythonSignal { //Negative Scenarios if (signalnum < 1 || signalnum > 22) { throw PythonOps.ValueError("signal number out of range"); - } else if (!GetPythonSignalState(context).PySignalToPyHandler.ContainsKey(signalnum)) { + } else if (GetPythonSignalState(context).PySignalToPyHandler.TryGetValue(signalnum, out object value)) { + //Default + return value; + } else { //Handles the special case of SIG_IGN. This is not really a signal, //but CPython returns null for it any ways return null; - } else { - //Default - return GetPythonSignalState(context).PySignalToPyHandler[signalnum]; } } } diff --git a/Src/IronPython.Modules/_ssl.cs b/Src/IronPython.Modules/_ssl.cs index 95db1d391..afd49f4ea 100644 --- a/Src/IronPython.Modules/_ssl.cs +++ b/Src/IronPython.Modules/_ssl.cs @@ -205,7 +205,7 @@ public class _SSLContext { public PythonList get_ca_certs(CodeContext context, bool binary_form = false) { if (binary_form) throw new NotImplementedException(nameof(binary_form)); - return new PythonList(_cert_store.Cast().Select(c => CertificateToPython(context, c))); + return PythonList.FromEnumerable(_cert_store.Cast().Select(c => CertificateToPython(context, c))); } public void load_verify_locations(CodeContext context, object cafile = null, string capath = null, object cadata = null) { diff --git a/Src/IronPython.Modules/grp.cs b/Src/IronPython.Modules/grp.cs index daa42c770..050bac7b7 100644 --- a/Src/IronPython.Modules/grp.cs +++ b/Src/IronPython.Modules/grp.cs @@ -77,7 +77,7 @@ public class struct_group : PythonTuple { private static struct_group Make(IntPtr pwd) { group g = (group)Marshal.PtrToStructure(pwd, typeof(group)); - return new struct_group(g.gr_name, g.gr_passwd, g.gr_gid, new PythonList(MarshalStringArray(g.gr_mem))); + return new struct_group(g.gr_name, g.gr_passwd, g.gr_gid, PythonList.FromEnumerable(MarshalStringArray(g.gr_mem))); } private static IEnumerable MarshalStringArray(IntPtr arrayPtr) { diff --git a/Src/IronPython.Modules/nt.cs b/Src/IronPython.Modules/nt.cs index 4ceb42b49..ff50a61e9 100644 --- a/Src/IronPython.Modules/nt.cs +++ b/Src/IronPython.Modules/nt.cs @@ -1829,13 +1829,12 @@ public static void utime(CodeContext context, object? path, [ParamDictionary, No public static int write(CodeContext/*!*/ context, int fd, [NotNone] IBufferProtocol data) { try { + using var buffer = data.GetBuffer(); PythonContext pythonContext = context.LanguageContext; var streams = pythonContext.FileManager.GetStreams(fd); - using var buffer = data.GetBuffer(); - var bytes = buffer.AsReadOnlySpan(); if (!streams.WriteStream.CanWrite) throw PythonOps.OSError(9, "Bad file descriptor"); - return streams.Write(bytes); + return streams.Write(buffer); } catch (Exception e) { throw ToPythonException(e); } diff --git a/Src/IronPython.Modules/time.cs b/Src/IronPython.Modules/time.cs index f93f18427..806efd26c 100644 --- a/Src/IronPython.Modules/time.cs +++ b/Src/IronPython.Modules/time.cs @@ -17,6 +17,7 @@ using System.Threading; using IronPython.Runtime; +using IronPython.Runtime.Exceptions; using IronPython.Runtime.Operations; using IronPython.Runtime.Types; @@ -106,18 +107,31 @@ public static string asctime(CodeContext/*!*/ context, [NotNone] PythonTuple tim public static double perf_counter() => clock(); + public static double process_time() => Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; + public static string ctime(CodeContext/*!*/ context) => asctime(context, localtime()); public static string ctime(CodeContext/*!*/ context, object? seconds) => asctime(context, localtime(seconds)); - public static object get_clock_info([NotNone] string name) { + public static object get_clock_info(CodeContext/*!*/ context, [NotNone] string name) { // TODO: Fill with correct values - if (name == "monotonic") - return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.GetTimestamp" }, { "monotonic", true }, { "resolution", 0.015625 } }); - - throw new NotImplementedException(); + switch (name) { + case "monotonic": + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.GetTimestamp" }, { "monotonic", true }, { "resolution", 0.015625 } }); + case "perf_counter": + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.ElapsedTicks" }, { "monotonic", true }, { "resolution", 1e-7 } }); + case "process_time": + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Process.TotalProcessorTime" }, { "monotonic", true }, { "resolution", 1e-7 } }); + case "clock": + PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead"); + return new SimpleNamespace(new Dictionary { { "adjustable", false }, { "implementation", "Stopwatch.ElapsedTicks" }, { "monotonic", true }, { "resolution", 1e-7 } }); + case "time": + return new SimpleNamespace(new Dictionary { { "adjustable", true }, { "implementation", "DateTime.Now" }, { "monotonic", false }, { "resolution", 0.015625 } }); + default: + throw PythonOps.ValueError("unknown clock"); + } } public static void sleep(double tm) { diff --git a/Src/IronPython.Modules/xxsubtype.cs b/Src/IronPython.Modules/xxsubtype.cs index 031e092e8..8980fc8c1 100644 --- a/Src/IronPython.Modules/xxsubtype.cs +++ b/Src/IronPython.Modules/xxsubtype.cs @@ -24,8 +24,8 @@ public spamlist() : base() { } - public spamlist(object sequence) - : base(sequence) { + public spamlist(CodeContext context, object sequence) + : base(context, sequence) { } private int _state; diff --git a/Src/IronPython.SQLite/Cursor.cs b/Src/IronPython.SQLite/Cursor.cs index 772429d7c..8570ad7ef 100644 --- a/Src/IronPython.SQLite/Cursor.cs +++ b/Src/IronPython.SQLite/Cursor.cs @@ -121,7 +121,7 @@ private object queryExecute(CodeContext context, bool multiple, object operation if(multiple) { if(args != null) - parameters_iter = PythonOps.CreatePythonEnumerator(args); + parameters_iter = PythonOps.CreatePythonEnumerator(context, args); } else { diff --git a/Src/IronPython.SQLite/IronPython.SQLite.csproj b/Src/IronPython.SQLite/IronPython.SQLite.csproj index 95e250e07..f00505312 100644 --- a/Src/IronPython.SQLite/IronPython.SQLite.csproj +++ b/Src/IronPython.SQLite/IronPython.SQLite.csproj @@ -1,12 +1,18 @@  - net462;netcoreapp3.1;netstandard2.0;net6.0 + net462;netstandard2.0;net6.0 true SQLITE_DEBUG;TRUE;WIN32;_MSC_VER;SQLITE_ASCII;SQLITE_MEM_POOL;SQLITE_ENABLE_COLUMN_METADATA;SQLITE_OS_WIN;SQLITE_SYSTEM_MALLOC;VDBE_PROFILE_OFF SQLITE_OMIT_AUTHORIZATION;SQLITE_OMIT_DEPRECATED;SQLITE_OMIT_GET_TABLE;SQLITE_OMIT_INCRBLOB;SQLITE_OMIT_LOOKASIDE;SQLITE_OMIT_SHARED_CACHE;SQLITE_OMIT_UTF16;SQLITE_OMIT_WAL $(NoWarn);0168;0169;0414;0618;0649;1587;219;1570 true + + + false + $(BaseIntermediateOutputPath)$(Configuration)\$(TargetFramework)\DLLs + $(BaseOutputPath)\$(TargetFramework)\DLLs + true @@ -18,7 +24,9 @@ - + + + diff --git a/Src/IronPython.Wpf/IronPython.Wpf.csproj b/Src/IronPython.Wpf/IronPython.Wpf.csproj index 23b38c9c6..9d0ff5997 100644 --- a/Src/IronPython.Wpf/IronPython.Wpf.csproj +++ b/Src/IronPython.Wpf/IronPython.Wpf.csproj @@ -1,14 +1,21 @@  - net462;netcoreapp3.1;net6.0-windows + net462;net6.0-windows true true true + + + false + $(BaseOutputPath)\$(TargetFramework.Replace('-windows', ''))\DLLs + true - + + + diff --git a/Src/IronPython/Hosting/PythonCommandLine.cs b/Src/IronPython/Hosting/PythonCommandLine.cs index 2df72148a..3d1a9fee6 100644 --- a/Src/IronPython/Hosting/PythonCommandLine.cs +++ b/Src/IronPython/Hosting/PythonCommandLine.cs @@ -36,9 +36,9 @@ public class PythonCommandLine : CommandLine { protected override string Logo => PythonContext.PythonOptions.Quiet ? null : GetLogoDisplay(); /// - /// Returns the display look for IronPython. - /// - /// The returned string uses This \n instead of Environment.NewLine for it's line seperator + /// Returns the display look for IronPython. + /// + /// The returned string uses This \n instead of Environment.NewLine for it's line seperator /// because it is intended to be outputted through the Python I/O system. /// public static string GetLogoDisplay() { @@ -265,6 +265,7 @@ public class PythonCommandLine : CommandLine { if (File.Exists(runner)) { executable = runner; } else { + // TODO: was for .NET Core 2.1, can we drop this? runner = Path.Combine(prefix, name + ".bat"); if (File.Exists(runner)) executable = runner; } @@ -312,7 +313,7 @@ public class PythonCommandLine : CommandLine { /// /// Loads any extension DLLs present in sys.prefix\DLLs directory and adds references to them. - /// + /// /// This provides an easy drop-in location for .NET assemblies which should be automatically referenced /// (exposed via import), COM libraries, and pre-compiled Python code. /// @@ -433,7 +434,7 @@ public class PythonCommandLine : CommandLine { /// Attempts to run a single interaction and handle any language-specific /// exceptions. Base classes can override this and call the base implementation /// surrounded with their own exception handling. - /// + /// /// Returns null if successful and execution should continue, or an exit code. /// private int? TryInteractiveActionWorker() { @@ -457,8 +458,8 @@ public class PythonCommandLine : CommandLine { } /// - /// Parses a single interactive command and executes it. - /// + /// Parses a single interactive command and executes it. + /// /// Returns null if successful and execution should continue, or the appropiate exit code. /// private int? RunOneInteraction() { diff --git a/Src/IronPython/IronPython.csproj b/Src/IronPython/IronPython.csproj index d86b1393a..b44389ed9 100644 --- a/Src/IronPython/IronPython.csproj +++ b/Src/IronPython/IronPython.csproj @@ -1,7 +1,7 @@  - net462;netcoreapp3.1;netstandard2.0;net6.0 + net462;netstandard2.0;net6.0 879755264 true true @@ -66,7 +66,7 @@ - + diff --git a/Src/IronPython/Modules/Builtin.cs b/Src/IronPython/Modules/Builtin.cs index fd0856968..c98617b89 100644 --- a/Src/IronPython/Modules/Builtin.cs +++ b/Src/IronPython/Modules/Builtin.cs @@ -250,7 +250,7 @@ public static partial class Builtin { public static PythonType dict => TypeCache.Dict; public static PythonList dir(CodeContext/*!*/ context) { - PythonList res = new PythonList(context.Dict.Keys); + PythonList res = new PythonList(context, context.Dict.Keys); res.Sort(context); return res; @@ -258,7 +258,7 @@ public static partial class Builtin { public static PythonList dir(CodeContext/*!*/ context, object? o) { IList ret = PythonOps.GetAttrNames(context, o); - PythonList lret = new PythonList(ret); + PythonList lret = new PythonList(context, ret); lret.Sort(context); return lret; } diff --git a/Src/IronPython/Modules/_ast.cs b/Src/IronPython/Modules/_ast.cs index 679e1e70e..42d5deabc 100755 --- a/Src/IronPython/Modules/_ast.cs +++ b/Src/IronPython/Modules/_ast.cs @@ -1773,7 +1773,7 @@ public Global(PythonList names, [Optional] int? lineno, [Optional] int? col_offs internal Global(GlobalStatement stmt) : this() { - names = new PythonList(stmt.Names); + names = PythonList.FromGenericCollection(stmt.Names); } internal override Statement Revert() { @@ -2345,7 +2345,7 @@ public Nonlocal(PythonList names, [Optional] int? lineno, [Optional] int? col_of internal Nonlocal(NonlocalStatement stmt) : this() { - names = new PythonList(stmt.Names); + names = PythonList.FromGenericCollection(stmt.Names); } internal override Statement Revert() { diff --git a/Src/IronPython/Modules/_fileio.cs b/Src/IronPython/Modules/_fileio.cs index b0072caf7..6f077efef 100644 --- a/Src/IronPython/Modules/_fileio.cs +++ b/Src/IronPython/Modules/_fileio.cs @@ -354,14 +354,7 @@ public FileIO(CodeContext/*!*/ context, string name, string mode = "r", bool clo _checkClosed(); - var span = pythonBuffer.AsSpan(); - for (int i = 0; i < span.Length; i++) { - int b = _streams.ReadStream.ReadByte(); - if (b == -1) return i; - span[i] = (byte)b; - } - - return span.Length; + return _streams.ReadInto(pythonBuffer); } public override BigInteger readinto(CodeContext/*!*/ context, object buf) { @@ -447,10 +440,9 @@ public FileIO(CodeContext/*!*/ context, string name, string mode = "r", bool clo public override BigInteger write(CodeContext/*!*/ context, object b) { var bufferProtocol = Converter.Convert(b); using var buffer = bufferProtocol.GetBuffer(); - var bytes = buffer.AsReadOnlySpan(); EnsureWritable(); - return _streams.Write(bytes); + return _streams.Write(buffer); } #endregion diff --git a/Src/IronPython/Runtime/Binding/ConversionBinder.cs b/Src/IronPython/Runtime/Binding/ConversionBinder.cs index 507e395a6..61349b7aa 100644 --- a/Src/IronPython/Runtime/Binding/ConversionBinder.cs +++ b/Src/IronPython/Runtime/Binding/ConversionBinder.cs @@ -781,7 +781,7 @@ private class IdentityConversion { PythonTypeSlot pts; if (pt.TryResolveSlot(context, "__iter__", out pts)) { - return MakeIterRule(metaUserObject, nameof(PythonOps.CreatePythonEnumerable)); + return MakeIterRule(metaUserObject, pyContext, nameof(PythonOps.CreatePythonEnumerable)); } else if (pt.TryResolveSlot(context, "__getitem__", out pts)) { return MakeGetItemIterable(metaUserObject, pyContext, pts, nameof(PythonOps.CreateItemEnumerable)); } @@ -804,6 +804,7 @@ private class IdentityConversion { new[] { tmp }, Expression.Call( typeof(PythonOps).GetMethod(nameof(PythonOps.CreatePythonEnumerator)), + AstUtils.Constant(context), Ast.Block( MetaPythonObject.MakeTryGetTypeMember( state, @@ -839,6 +840,7 @@ private class IdentityConversion { new[] { tmp }, Expression.Call( typeof(PythonOps).GetMethod(method), + AstUtils.Constant(state.SharedContext), AstUtils.Convert(metaUserObject.Expression, typeof(object)), Ast.Block( MetaPythonObject.MakeTryGetTypeMember( @@ -867,10 +869,11 @@ private class IdentityConversion { ); } - private static DynamicMetaObject/*!*/ MakeIterRule(DynamicMetaObject/*!*/ self, string methodName) { + private static DynamicMetaObject/*!*/ MakeIterRule(DynamicMetaObject/*!*/ self, PythonContext state, string methodName) { return new DynamicMetaObject( Ast.Call( typeof(PythonOps).GetMethod(methodName), + AstUtils.Constant(state.SharedContext), AstUtils.Convert(self.Expression, typeof(object)) ), self.Restrictions diff --git a/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs b/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs index 245afa076..cd492f595 100644 --- a/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs +++ b/Src/IronPython/Runtime/Binding/MetaPythonFunction.cs @@ -955,6 +955,7 @@ private class FunctionBinderHelper { _params, Ast.Call( typeof(PythonOps).GetMethod(nameof(PythonOps.CopyAndVerifyParamsList)), + _codeContext ?? AstUtils.Constant(DefaultContext.Default), AstUtils.Convert(GetFunctionParam(), typeof(PythonFunction)), AstUtils.Convert(userList, typeof(object)) ) diff --git a/Src/IronPython/Runtime/BufferProtocol.cs b/Src/IronPython/Runtime/BufferProtocol.cs index 424055402..f537ae709 100644 --- a/Src/IronPython/Runtime/BufferProtocol.cs +++ b/Src/IronPython/Runtime/BufferProtocol.cs @@ -308,23 +308,56 @@ public static BufferEnumerator EnumerateItemData(this IPythonBuffer buffer) } } + /// + /// Obtain the underlying array, if possible. + /// The returned array is unsafe because it should not be written to. + /// internal static byte[]? AsUnsafeArray(this IPythonBuffer buffer) { if (!buffer.IsCContiguous()) return null; - if (buffer.Object is Bytes b) - return b.UnsafeByteArray; - - if (buffer.Object is Memory mem) { - if (MemoryMarshal.TryGetArray(mem, out ArraySegment seg) && seg.Array != null && seg.Offset == 0 && seg.Count == seg.Array.Length) + ReadOnlySpan bufdata = buffer.AsReadOnlySpan(); + if (buffer.Object is Bytes b) { + if (b.UnsafeByteArray.AsSpan() == bufdata) + return b.UnsafeByteArray; + } else if (buffer.Object is ByteArray ba) { + byte[] arrdata = ba.UnsafeByteList.Data; + if (arrdata.AsSpan() == bufdata) + return arrdata; + } else if (buffer.Object is Memory mem) { + if (MemoryMarshal.TryGetArray(mem, out ArraySegment seg) && seg.Array is not null && seg.Array.AsSpan() == bufdata) return seg.Array; } else if (buffer.Object is ReadOnlyMemory rom) { - if (MemoryMarshal.TryGetArray(rom, out ArraySegment seg) && seg.Array != null && seg.Offset == 0 && seg.Count == seg.Array.Length) + if (MemoryMarshal.TryGetArray(rom, out ArraySegment seg) && seg.Array is not null && seg.Array.AsSpan() == bufdata) + return seg.Array; + } + + return null; + } + + /// + /// Obtain the underlying writable array, if possible. + /// The returned array is unsafe because it can be longer than the buffer. + /// + internal static byte[]? AsUnsafeWritableArray(this IPythonBuffer buffer) { + if (!buffer.IsCContiguous() || buffer.IsReadOnly) + return null; + + Span bufdata = buffer.AsSpan(); + if (buffer.Object is ByteArray ba) { + byte[] arrdata = ba.UnsafeByteList.Data; + if (UseSameMemory(arrdata, bufdata)) + return arrdata; + } else if (buffer.Object is Memory mem) { + if (MemoryMarshal.TryGetArray(mem, out ArraySegment seg) && seg.Array is not null && UseSameMemory(seg.Array, bufdata)) return seg.Array; } return null; } + + private static bool UseSameMemory(byte[] arr, ReadOnlySpan span) + => arr.Length >= span.Length && arr.AsSpan(0, span.Length) == span; } public ref struct BufferBytesEnumerator { diff --git a/Src/IronPython/Runtime/Bytes.cs b/Src/IronPython/Runtime/Bytes.cs index e21706a57..282a93e47 100644 --- a/Src/IronPython/Runtime/Bytes.cs +++ b/Src/IronPython/Runtime/Bytes.cs @@ -1170,7 +1170,7 @@ public Bytes __rmod__(CodeContext context, [NotNone] Bytes value) System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { // workaround for https://github.com/IronLanguages/ironpython3/issues/1519 if (GetType() != typeof(Bytes) && PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, this, "__iter__", out object? iter)) { - return new PythonEnumerator(iter); + return new PythonEnumerator(DefaultContext.Default, iter); } return _bytes.GetEnumerator(); } diff --git a/Src/IronPython/Runtime/ClrModule.cs b/Src/IronPython/Runtime/ClrModule.cs index 0930d6620..7e7b404bc 100644 --- a/Src/IronPython/Runtime/ClrModule.cs +++ b/Src/IronPython/Runtime/ClrModule.cs @@ -837,9 +837,9 @@ public RuntimeReturnChecker(object instance, [NotNone] object function, [NotNone /// /// returns the result of dir(o) as-if "import clr" has not been performed. /// - public static PythonList Dir(object o) { + public static PythonList Dir(CodeContext context, object o) { IList ret = PythonOps.GetAttrNames(DefaultContext.Default, o); - PythonList lret = new PythonList(ret); + PythonList lret = new PythonList(context, ret); lret.Sort(DefaultContext.Default); return lret; } @@ -847,9 +847,9 @@ public RuntimeReturnChecker(object instance, [NotNone] object function, [NotNone /// /// Returns the result of dir(o) as-if "import clr" has been performed. /// - public static PythonList DirClr(object o) { + public static PythonList DirClr(CodeContext context, object o) { IList ret = PythonOps.GetAttrNames(DefaultContext.DefaultCLS, o); - PythonList lret = new PythonList(ret); + PythonList lret = new PythonList(context, ret); lret.Sort(DefaultContext.DefaultCLS); return lret; } diff --git a/Src/IronPython/Runtime/ConversionWrappers.cs b/Src/IronPython/Runtime/ConversionWrappers.cs index ef36804c7..2a61dfd85 100644 --- a/Src/IronPython/Runtime/ConversionWrappers.cs +++ b/Src/IronPython/Runtime/ConversionWrappers.cs @@ -123,10 +123,6 @@ public class ListGenericWrapper : IList { #endregion } -#if NETCOREAPP3_1 // In netcoreapp3.1, IDictionary has constraint where K : notnull -#nullable disable warnings -#endif - public class DictionaryGenericWrapper : IDictionary { private readonly IDictionary self; // PEP 237: int/long unification (GH #52) @@ -303,10 +299,6 @@ public class DictionaryGenericWrapper : IDictionary { } } -#if NETCOREAPP3_1 -#nullable enable -#endif - public class IEnumeratorOfTWrapper : IEnumerator { private readonly IEnumerator enumerable; // PEP 237: int/long unification (GH #52) @@ -327,7 +319,7 @@ public class IEnumeratorOfTWrapper : IEnumerator { try { return (T)current!; } catch (NullReferenceException nex) { - throw new InvalidCastException(string.Format("Error in IEnumeratorOfTWrapper.Current. Could not cast: from null to {0}", typeof(T).ToString()), nex); + throw new InvalidCastException(string.Format("Error in IEnumeratorOfTWrapper.Current. Could not cast: from null to {0}", typeof(T).ToString()), nex); } catch (InvalidCastException iex) { throw new InvalidCastException(string.Format("Error in IEnumeratorOfTWrapper.Current. Could not cast: from {1} to {0}", typeof(T).ToString(), current!.GetType().ToString()), iex); } diff --git a/Src/IronPython/Runtime/Enumerate.cs b/Src/IronPython/Runtime/Enumerate.cs index 654402cf9..c8fae3449 100644 --- a/Src/IronPython/Runtime/Enumerate.cs +++ b/Src/IronPython/Runtime/Enumerate.cs @@ -28,14 +28,14 @@ public class Enumerate : IEnumerator, IEnumerator { private readonly IEnumerator _iter; private object _index; - public Enumerate(object iter) { - _iter = PythonOps.GetEnumerator(iter); + public Enumerate(CodeContext context, object iter) { + _iter = PythonOps.GetEnumerator(context, iter); _index = ScriptingRuntimeHelpers.Int32ToObject(-1); } public Enumerate(CodeContext context, object iter, object start) { object index = PythonOps.Index(start); - _iter = PythonOps.GetEnumerator(iter); + _iter = PythonOps.GetEnumerator(context, iter); _index = context.LanguageContext.Operation(Binding.PythonOperationKind.Subtract, index, ScriptingRuntimeHelpers.Int32ToObject(1)); } @@ -176,16 +176,17 @@ public sealed class SentinelIterator : IEnumerator, IEnumerator { [PythonType("enumerator")] public class PythonEnumerator : IEnumerator { + private readonly CodeContext _context; private readonly object _baseObject; private object _current; public static bool TryCastIEnumer(object baseObject, out IEnumerator enumerator) { - if (baseObject is IEnumerator) { - enumerator = (IEnumerator)baseObject; + if (baseObject is IEnumerator et) { + enumerator = et; return true; } - if (baseObject is IEnumerable) { - enumerator = ((IEnumerable)baseObject).GetEnumerator(); + if (baseObject is IEnumerable en) { + enumerator = en.GetEnumerator(); return true; } @@ -193,18 +194,18 @@ public class PythonEnumerator : IEnumerator { return false; } - public static bool TryCreate(object baseObject, out IEnumerator enumerator) { + public static bool TryCreate(CodeContext context, object baseObject, out IEnumerator enumerator) { if (TryCastIEnumer(baseObject, out enumerator)) { return true; } - if (PythonOps.TryGetBoundAttr(baseObject, "__iter__", out object iter)) { + if (PythonOps.TryGetBoundAttr(context, baseObject, "__iter__", out object iter)) { object iterator = PythonCalls.Call(iter); // don't re-wrap if we don't need to (common case is PythonGenerator). if (TryCastIEnumer(iterator, out enumerator)) { return true; } - enumerator = new PythonEnumerator(iterator); + enumerator = new PythonEnumerator(context, iterator); return true; } else { enumerator = null; @@ -212,17 +213,18 @@ public class PythonEnumerator : IEnumerator { } } - public static IEnumerator Create(object baseObject) { + public static IEnumerator Create(CodeContext context, object baseObject) { IEnumerator res; - if (!TryCreate(baseObject, out res)) { + if (!TryCreate(context, baseObject, out res)) { throw PythonOps.TypeError("cannot convert {0} to IEnumerator", PythonOps.GetPythonTypeName(baseObject)); } return res; } - internal PythonEnumerator(object iter) { + internal PythonEnumerator(CodeContext context, object iter) { Debug.Assert(!(iter is PythonGenerator)); + _context = context; _baseObject = iter; } @@ -244,14 +246,14 @@ public class PythonEnumerator : IEnumerator { /// /// True if moving was successfull public bool MoveNext() { - PythonTypeOps.TryGetOperator(DefaultContext.Default, _baseObject, "__next__", out object nextMethod); + PythonTypeOps.TryGetOperator(_context, _baseObject, "__next__", out object nextMethod); if (nextMethod == null) { throw PythonOps.TypeErrorForNotAnIterator(_baseObject); } try { - _current = DefaultContext.Default.LanguageContext.CallLightEh(DefaultContext.Default, nextMethod); + _current = _context.LanguageContext.CallLightEh(_context, nextMethod); Exception lightEh = LightExceptions.GetLightException(_current); if (lightEh != null) { if (lightEh is StopIterationException) { @@ -275,21 +277,22 @@ public class PythonEnumerator : IEnumerator { [PythonType("enumerable")] public class PythonEnumerable : IEnumerable { + private readonly CodeContext _context; private readonly object _iterator; - public static bool TryCreate(object baseEnumerator, out IEnumerable enumerator) { + public static bool TryCreate(CodeContext context, object baseEnumerator, out IEnumerable enumerator) { Debug.Assert(!(baseEnumerator is IEnumerable) || baseEnumerator is IPythonObject); // we shouldn't re-wrap things that don't need it - if (PythonOps.TryGetBoundAttr(baseEnumerator, "__iter__", out object iter)) { - object iterator = PythonCalls.Call(iter); - if (iterator is IEnumerable) { - enumerator = (IEnumerable)iterator; + if (PythonOps.TryGetBoundAttr(context, baseEnumerator, "__iter__", out object iter)) { + object iterator = PythonCalls.Call(context, iter); + if (iterator is IEnumerable en) { + enumerator = en; } else { - if (!PythonOps.TryGetBoundAttr(iterator, "__next__", out _)) { + if (!PythonOps.TryGetBoundAttr(context, iterator, "__next__", out _)) { enumerator = null; return false; } - enumerator = new PythonEnumerable(iterator); + enumerator = new PythonEnumerable(context, iterator); } return true; } else { @@ -298,22 +301,23 @@ public class PythonEnumerable : IEnumerable { } } - public static IEnumerable Create(object baseObject) { + public static IEnumerable Create(CodeContext context, object baseObject) { IEnumerable res; - if (!TryCreate(baseObject, out res)) { + if (!TryCreate(context, baseObject, out res)) { throw PythonOps.TypeError("cannot convert {0} to IEnumerable", PythonOps.GetPythonTypeName(baseObject)); } return res; } - private PythonEnumerable(object iterator) { - this._iterator = iterator; + private PythonEnumerable(CodeContext context, object iterator) { + _iterator = iterator; + _context = iterator is IEnumerable ? DefaultContext.Default : context; } #region IEnumerable Members IEnumerator IEnumerable.GetEnumerator() { - return _iterator as IEnumerator ?? new PythonEnumerator(_iterator); + return _iterator as IEnumerator ?? new PythonEnumerator(_context, _iterator); } #endregion @@ -321,6 +325,7 @@ public class PythonEnumerable : IEnumerable { [PythonType("iterator")] public sealed class ItemEnumerator : IEnumerator { + private readonly CodeContext _context; // The actual object on which we are calling __getitem__() private object _source; private object _getItemMethod; @@ -328,7 +333,8 @@ public sealed class ItemEnumerator : IEnumerator { private object _current; private int _index; - internal ItemEnumerator(object source, object getItemMethod, CallSite> site) { + internal ItemEnumerator(CodeContext context, object source, object getItemMethod, CallSite> site) { + _context = context; _source = source; _getItemMethod = getItemMethod; _site = site; @@ -374,7 +380,7 @@ public sealed class ItemEnumerator : IEnumerator { } try { - _current = _site.Target(_site, DefaultContext.Default, _getItemMethod, _index); + _current = _site.Target(_site, _context, _getItemMethod, _index); _index++; return true; } catch (IndexOutOfRangeException) { @@ -404,11 +410,13 @@ public sealed class ItemEnumerator : IEnumerator { [PythonType("iterable")] public sealed class ItemEnumerable : IEnumerable { + private readonly CodeContext _context; private readonly object _source; private readonly object _getitem; private readonly CallSite> _site; - internal ItemEnumerable(object source, object getitem, CallSite> site) { + internal ItemEnumerable(CodeContext context, object source, object getitem, CallSite> site) { + _context = context; _source = source; _getitem = getitem; _site = site; @@ -421,7 +429,7 @@ public sealed class ItemEnumerable : IEnumerable { #region IEnumerable Members IEnumerator IEnumerable.GetEnumerator() { - return new ItemEnumerator(_source, _getitem, _site); + return new ItemEnumerator(_context, _source, _getitem, _site); } #endregion diff --git a/Src/IronPython/Runtime/Operations/InstanceOps.cs b/Src/IronPython/Runtime/Operations/InstanceOps.cs index 52c018892..2b6c6578f 100644 --- a/Src/IronPython/Runtime/Operations/InstanceOps.cs +++ b/Src/IronPython/Runtime/Operations/InstanceOps.cs @@ -212,7 +212,7 @@ public static class InstanceOps { /// __dir__(self) -> Returns the list of members defined on a foreign IDynamicMetaObjectProvider. /// public static PythonList DynamicDir(CodeContext/*!*/ context, IDynamicMetaObjectProvider self) { - PythonList res = new PythonList(self.GetMetaObject(Expression.Parameter(typeof(object))).GetDynamicMemberNames()); + PythonList res = new PythonList(context, self.GetMetaObject(Expression.Parameter(typeof(object))).GetDynamicMemberNames()); // add in the non-dynamic members from the dynamic objects base class. Type t = self.GetType(); diff --git a/Src/IronPython/Runtime/Operations/PythonOps.cs b/Src/IronPython/Runtime/Operations/PythonOps.cs index 6e57eb516..31397c8b3 100644 --- a/Src/IronPython/Runtime/Operations/PythonOps.cs +++ b/Src/IronPython/Runtime/Operations/PythonOps.cs @@ -1084,7 +1084,7 @@ internal static string FsPathDecoded(CodeContext context, object? path) } if (o is IMembersList memList) { - return new PythonList(memList.GetMemberNames()); + return new PythonList(context, memList.GetMemberNames()); } if (o is IPythonObject po) { @@ -1782,7 +1782,7 @@ internal static string FsPathDecoded(CodeContext context, object? path) /// LIST_EXTEND /// [EditorBrowsable(EditorBrowsableState.Never)] - public static void ListExtend(PythonList list, object? o) => list.extend(o); + public static void ListExtend(PythonList list, object? o) => list.extend(DefaultContext.Default, o); /// /// LIST_TO_TUPLE @@ -2655,8 +2655,8 @@ internal static void PrintWithDestNoNewline(CodeContext/*!*/ context, object des } - public static PythonList CopyAndVerifyParamsList(PythonFunction function, object list) { - return new PythonList(list); + public static PythonList CopyAndVerifyParamsList(CodeContext context, PythonFunction function, object list) { + return new PythonList(context, list); } public static PythonTuple UserMappingToPythonTuple(CodeContext/*!*/ context, object list, string funcName) { @@ -3259,24 +3259,24 @@ internal static void PrintWithDestNoNewline(CodeContext/*!*/ context, object des return ((PythonGenerator)self).CheckThrowableAndReturnSendValue(); } - public static ItemEnumerable CreateItemEnumerable(object source, object callable, CallSite> site) { - return new ItemEnumerable(source, callable, site); + public static ItemEnumerable CreateItemEnumerable(CodeContext context, object source, object callable, CallSite> site) { + return new ItemEnumerable(context, source, callable, site); } public static DictionaryKeyEnumerator MakeDictionaryKeyEnumerator(PythonDictionary dict) { return new DictionaryKeyEnumerator(dict._storage); } - public static IEnumerable CreatePythonEnumerable(object baseObject) { - return PythonEnumerable.Create(baseObject); + public static IEnumerable CreatePythonEnumerable(CodeContext context, object baseObject) { + return PythonEnumerable.Create(context, baseObject); } - public static IEnumerator CreateItemEnumerator(object source, object callable, CallSite> site) { - return new ItemEnumerator(source, callable, site); + public static IEnumerator CreateItemEnumerator(CodeContext context, object source, object callable, CallSite> site) { + return new ItemEnumerator(context, source, callable, site); } - public static IEnumerator CreatePythonEnumerator(object baseObject) { - return PythonEnumerator.Create(baseObject); + public static IEnumerator CreatePythonEnumerator(CodeContext context, object baseObject) { + return PythonEnumerator.Create(context, baseObject); } public static bool ContainsFromEnumerable(CodeContext/*!*/ context, object enumerable, object value) { diff --git a/Src/IronPython/Runtime/Operations/StringOps.cs b/Src/IronPython/Runtime/Operations/StringOps.cs index 74f041661..f79855b95 100644 --- a/Src/IronPython/Runtime/Operations/StringOps.cs +++ b/Src/IronPython/Runtime/Operations/StringOps.cs @@ -2091,7 +2091,7 @@ internal static partial class CodecsInfo { #if DEBUG foreach (KeyValuePair> kvp in d) { // all codecs should be stored in lowercase because we only look up from lowercase strings - Debug.Assert(kvp.Key.ToLower(CultureInfo.InvariantCulture) == kvp.Key); + Debug.Assert(kvp.Key.Equals(kvp.Key, StringComparison.OrdinalIgnoreCase)); // all codec names should use underscores instead of dashes to match lookup values Debug.Assert(kvp.Key.IndexOf('-') < 0); } diff --git a/Src/IronPython/Runtime/PythonContext.cs b/Src/IronPython/Runtime/PythonContext.cs index 0e17d7633..40b62180e 100644 --- a/Src/IronPython/Runtime/PythonContext.cs +++ b/Src/IronPython/Runtime/PythonContext.cs @@ -1261,7 +1261,7 @@ private class AssemblyResolveHolder { public override ICollection GetSearchPaths() { List result = new List(); if (TryGetSystemPath(out PythonList paths)) { - IEnumerator ie = PythonOps.GetEnumerator(paths); + IEnumerator ie = PythonOps.GetEnumerator(SharedContext, paths); while (ie.MoveNext()) { if (TryConvertToString(ie.Current, out string str)) { result.Add(str); @@ -1272,7 +1272,7 @@ private class AssemblyResolveHolder { } public override void SetSearchPaths(ICollection paths) { - SetSystemStateValue("path", new PythonList(paths)); + SetSystemStateValue("path", new PythonList(SharedContext, paths)); } public override void Shutdown() { diff --git a/Src/IronPython/Runtime/PythonFileManager.cs b/Src/IronPython/Runtime/PythonFileManager.cs index f5c3c9aa8..5070b9ffa 100644 --- a/Src/IronPython/Runtime/PythonFileManager.cs +++ b/Src/IronPython/Runtime/PythonFileManager.cs @@ -5,6 +5,7 @@ #nullable enable using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; @@ -109,17 +110,48 @@ internal sealed class StreamBox { return buffer; } - public int Write(ReadOnlySpan bytes) { + public int ReadInto(IPythonBuffer buffer) { #if NETCOREAPP + return _readStream.Read(buffer.AsSpan()); +#else + byte[]? bytes = buffer.AsUnsafeWritableArray(); + if (bytes is not null) { + return _readStream.Read(bytes, 0, buffer.NumBytes()); + } + + var span = buffer.AsSpan(); + const int chunkSize = 0x1000; // 4 KiB, default buffer size of FileSteam + bytes = ArrayPool.Shared.Rent(chunkSize); + try { + for (int pos = 0; pos < span.Length; pos += chunkSize) { + int toRead = Math.Min(chunkSize, span.Length - pos); + int hasRead = _readStream.Read(bytes, 0, toRead); + bytes.AsSpan(0, hasRead).CopyTo(span.Slice(pos)); + if (hasRead < toRead) return pos + hasRead; + } + } finally { + ArrayPool.Shared.Return(bytes); + } + return span.Length; +#endif + } + + public int Write(IPythonBuffer buffer) { + int count; +#if NETCOREAPP + ReadOnlySpan bytes = buffer.AsReadOnlySpan(); + count = bytes.Length; _writeStream.Write(bytes); #else - _writeStream.Write(bytes.ToArray(), 0, bytes.Length); + byte[] bytes = buffer.AsUnsafeArray() ?? buffer.AsUnsafeWritableArray() ?? buffer.ToArray(); + count = buffer.NumBytes(); + _writeStream.Write(bytes, 0, count); #endif _writeStream.Flush(); // IO at this level is not supposed to buffer so we need to call Flush. if (!IsSingleStream) { _readStream.Seek(_writeStream.Position, SeekOrigin.Begin); } - return bytes.Length; + return count; } public void Flush() { diff --git a/Src/IronPython/Runtime/PythonList.cs b/Src/IronPython/Runtime/PythonList.cs index 11ddb981c..dcf717b4e 100644 --- a/Src/IronPython/Runtime/PythonList.cs +++ b/Src/IronPython/Runtime/PythonList.cs @@ -33,6 +33,8 @@ public class PythonList : IList, ICodeFormattable, IList, IReversible, internal int _size; internal volatile object?[] _data; + #region Python Constructors and Initializers + public void __init__() { _data = new object[8]; _size = 0; @@ -106,7 +108,7 @@ public class PythonList : IList, ICodeFormattable, IList, IReversible, _data = new object[len]; _size = 0; - ExtendNoLengthCheck(sequence); + ExtendNoLengthCheck(context, sequence); } public static object __new__(CodeContext/*!*/ context, [NotNone] PythonType cls) { @@ -126,9 +128,30 @@ public static object __new__(CodeContext/*!*/ context, [NotNone] PythonType cls, public static object __new__(CodeContext/*!*/ context, [NotNone] PythonType cls, [ParamDictionary, NotNone] IDictionary kwArgs\u00F8, [NotNone] params object[] args\u00F8) => __new__(context, cls); - private PythonList(IEnumerator e) - : this(10) { - while (e.MoveNext()) AddNoLock(e.Current); + #endregion + + #region C# Constructors and Factories + + public PythonList() + : this(0) { + } + + public PythonList(CodeContext context, [NotNone] object sequence) { + if (sequence is ICollection items) { + _data = new object[items.Count]; + int i = 0; + foreach (object? item in items) { + _data[i++] = item; + } + _size = i; + } else { + if (!PythonOps.TryInvokeLengthHint(context, sequence, out int len)) { + len = INITIAL_SIZE; + } + + _data = new object[len]; + ExtendNoLengthCheck(context, sequence); + } } internal PythonList(int capacity) { @@ -139,13 +162,24 @@ private PythonList(IEnumerator e) } } - private PythonList(params object?[] items) { + internal PythonList(ICollection items) + : this(items.Count) { + + int i = 0; + foreach (object? item in items) { + _data[i++] = item; + } + _size = i; + } + + private PythonList(object?[] items) { _data = items; _size = _data.Length; } - public PythonList() - : this(0) { + private PythonList(IEnumerator e) + : this(10) { + while (e.MoveNext()) AddNoLock(e.Current); } #if ALLOC_DEBUG @@ -160,32 +194,24 @@ public PythonList() } #endif - internal PythonList(object sequence) { - if (sequence is ICollection items) { - _data = new object[items.Count]; - int i = 0; - foreach (object? item in items) { - _data[i++] = item; - } - _size = i; - } else { - if (!PythonOps.TryInvokeLengthHint(DefaultContext.Default, sequence, out int len)) { - len = INITIAL_SIZE; - } + internal static PythonList FromGenericCollection(ICollection items) { + var list = new PythonList(items.Count); - _data = new object[len]; - ExtendNoLengthCheck(sequence); + int i = 0; + foreach (object? item in items) { + list._data[i++] = item; } + list._size = i; + return list; } - internal PythonList(ICollection items) - : this(items.Count) { - - int i = 0; - foreach (object? item in items) { - _data[i++] = item; + internal static PythonList FromEnumerable(IEnumerable items) { + var enumerator = items.GetEnumerator(); + try { + return new PythonList(enumerator); + } finally { + (enumerator as IDisposable)?.Dispose(); } - _size = i; } /// @@ -197,6 +223,8 @@ internal PythonList(ICollection items) internal static PythonList FromArrayNoCopy(params object[] data) => new PythonList(data); + #endregion + internal object?[] GetObjectArray() { lock (this) { return ArrayOps.CopyArray(_data, _size); @@ -445,7 +473,7 @@ private static PythonList InPlaceMultiplyWorker(PythonList self, int count) set { if (slice.step != null && (!(slice.step is int) || !slice.step.Equals(_boxedOne))) { // try to assign back to self: make a copy first - if (this == value) value = new PythonList(value); + if (this == value) value = new PythonList((ICollection)value); if (ValueRequiresNoLocks(value)) { // we don't need to worry about lock ordering of accesses to the @@ -743,19 +771,27 @@ private void SliceAssignNoLock(int index, object? value) } } - public void extend(object? seq) { - if (PythonOps.TryInvokeLengthHint(DefaultContext.Default, seq, out int len)) { + public void extend(CodeContext context, object? seq) { + if (PythonOps.TryInvokeLengthHint(context, seq, out int len)) { // CPython proceeds without resizing if the length overflows if (int.MaxValue - len >= Count) { EnsureSize(Count + len); } } - ExtendNoLengthCheck(seq); + ExtendNoLengthCheck(context, seq); + } + + internal void ExtendNoLock(ICollection seq) { + EnsureSize(Count + seq.Count); + + foreach (var item in seq) { + AddNoLock(item); + } } - private void ExtendNoLengthCheck(object? seq) { - IEnumerator i = PythonOps.GetEnumerator(seq); + private void ExtendNoLengthCheck(CodeContext context, object? seq) { + IEnumerator i = PythonOps.GetEnumerator(context, seq); if (seq == (object)this) { PythonList other = new PythonList(i); i = ((IEnumerable)other).GetEnumerator(); diff --git a/Src/IronPython/Runtime/Set.cs b/Src/IronPython/Runtime/Set.cs index 487c7ec18..4fee48acd 100644 --- a/Src/IronPython/Runtime/Set.cs +++ b/Src/IronPython/Runtime/Set.cs @@ -1477,7 +1477,7 @@ public sealed class SetIterator : IEnumerable, IEnumerable, IEnumerator context.TryLookupBuiltin("iter", out iter); if (_cnt < 0) return PythonTuple.MakeTuple(iter, PythonTuple.MakeTuple(new PythonList())); - return PythonTuple.MakeTuple(iter, PythonTuple.MakeTuple(new PythonList(_items)), _cnt); + return PythonTuple.MakeTuple(iter, PythonTuple.MakeTuple(new PythonList(context, _items)), _cnt); } public int __length_hint__() { diff --git a/Src/IronPython/Runtime/Types/PythonType.cs b/Src/IronPython/Runtime/Types/PythonType.cs index 53688c208..6e774c591 100644 --- a/Src/IronPython/Runtime/Types/PythonType.cs +++ b/Src/IronPython/Runtime/Types/PythonType.cs @@ -1747,7 +1747,7 @@ public static PythonDictionary __prepare__([ParamDictionary]IDictionary strKeys = new List(keys.Keys); strKeys.Sort(); - res.extend(strKeys); + res.ExtendNoLock(strKeys); return res; } diff --git a/Src/IronPythonConsole/IronPythonConsole.csproj b/Src/IronPythonConsole/IronPythonConsole.csproj index 8bbde8148..73024a0c8 100644 --- a/Src/IronPythonConsole/IronPythonConsole.csproj +++ b/Src/IronPythonConsole/IronPythonConsole.csproj @@ -1,8 +1,8 @@  - net462;netcoreapp2.1;netcoreapp3.1;net6.0 - + net462;netcoreapp3.1;net6.0 + false Exe IronPythonConsole @@ -20,15 +20,12 @@ Content PreserveNewest - - - - - PreserveNewest - - + + + false + Content PreserveNewest - + @@ -38,7 +35,6 @@ - diff --git a/Src/IronPythonConsole/ipy.bat b/Src/IronPythonConsole/ipy.bat deleted file mode 100644 index 098af35bd..000000000 --- a/Src/IronPythonConsole/ipy.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -dotnet "%~dp0ipy.dll" %* diff --git a/Src/IronPythonTest/Cases/CaseExecuter.cs b/Src/IronPythonTest/Cases/CaseExecuter.cs index 3f5194efc..d75cbc047 100644 --- a/Src/IronPythonTest/Cases/CaseExecuter.cs +++ b/Src/IronPythonTest/Cases/CaseExecuter.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -31,8 +32,6 @@ internal class CaseExecuter { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { runner = Path.Combine(folder, "ipy.exe"); if (File.Exists(runner)) return runner; - runner = Path.Combine(folder, "ipy.bat"); - if (File.Exists(runner)) return runner; } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { runner = Path.Combine(folder, "ipy"); if (File.Exists(runner)) return runner; @@ -82,7 +81,7 @@ internal class CaseExecuter { private static void AddSearchPaths(ScriptEngine engine) { var paths = new List(engine.GetSearchPaths()); - if (!paths.Any(x => x.ToLowerInvariant().Contains("stdlib"))) { + if (!paths.Any(x => x.Contains("stdlib", StringComparison.OrdinalIgnoreCase))) { var root = FindRoot(); if (!string.IsNullOrEmpty(root)) { paths.Insert(0, Path.Combine(root, "Src", "StdLib", "Lib")); @@ -307,4 +306,12 @@ internal class CaseExecuter { } } } + +#if NETFRAMEWORK + internal static class StringExtensions { + public static bool Contains(this string s, string value, StringComparison comparisonType) { + return s.IndexOf(value, comparisonType) >= 0; + } + } +#endif } diff --git a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini index 66f410fc2..dca4d4f92 100644 --- a/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/IronPythonCasesManifest.ini @@ -201,9 +201,6 @@ RunCondition=NOT $(IS_OSX) # ctypes tests not prepared for macOS [IronPython.scripts.test_builder] Ignore=true -[IronPython.scripts.test_cgcheck] -Timeout=600000 # 10 minute timeout - [IronPython.scripts.test_parrot] Ignore=true diff --git a/Src/IronPythonTest/IronPythonTest.csproj b/Src/IronPythonTest/IronPythonTest.csproj index 3907d1cca..118756389 100644 --- a/Src/IronPythonTest/IronPythonTest.csproj +++ b/Src/IronPythonTest/IronPythonTest.csproj @@ -1,8 +1,8 @@  - net462;netcoreapp2.1;netcoreapp3.1;net6.0 - + net462;netcoreapp3.1;net6.0 + false true @@ -18,8 +18,8 @@ - - + + @@ -35,7 +35,7 @@ - + diff --git a/Src/IronPythonTest/Util/IniParser.cs b/Src/IronPythonTest/Util/IniParser.cs index a392dd2b2..a5101dfc1 100644 --- a/Src/IronPythonTest/Util/IniParser.cs +++ b/Src/IronPythonTest/Util/IniParser.cs @@ -72,7 +72,8 @@ public class IniParser { if (string.IsNullOrEmpty(line)) continue; - if (line.StartsWith("[", StringComparison.Ordinal) && line.EndsWith("]", StringComparison.Ordinal)) { + //if (line.StartsWith('[', StringComparison.Ordinal) && line.EndsWith(']', StringComparison.Ordinal)) { + if (line.Length >= 2 && line[0] == '[' && line[line.Length - 1] == ']') { var sectionName = line.Substring(1, line.Length - 2); if (!options.TryGetValue(sectionName, out currentSection)) { currentSection = new Section(); diff --git a/Src/Scripts/Install-IronPython.ps1 b/Src/Scripts/Install-IronPython.ps1 index 016596c42..030d93f3f 100755 --- a/Src/Scripts/Install-IronPython.ps1 +++ b/Src/Scripts/Install-IronPython.ps1 @@ -64,12 +64,13 @@ if (-not $ZipFile) { # Script run from within a checked out code base # Locate the zip archive in the standard location of the package target $projectRoot = $PSScriptRoot | Split-Path | Split-Path - $ZipFile = @(Resolve-Path (Join-Path $projectRoot "Package/Release/Packages/IronPython-*/IronPython.3.*.zip")) - if ($ZipFile.Count -gt 1) { - Write-Error "Ambiguous implicit project zip file: $ZipFile" - } elseif ($ZipFile.Count -lt 1) { + $zipFiles = @(Resolve-Path (Join-Path $projectRoot "Package/Release/Packages/IronPython-*/IronPython.3.*.zip")) + if ($zipFiles.Count -gt 1) { + Write-Error (@("Ambiguous implicit project zip files:") + $zipFiles -join "`n") + } elseif ($zipFiles.Count -lt 1) { Write-Error "Missing zip file. Have you run './make package'?" } + $ZipFile = $zipFiles } else { Write-Error "Cannot locate implicit zip file. Provide path to the zip file using '-ZipFile '." } @@ -98,7 +99,7 @@ if (-not $unzipDir) { } # Copy files into place -Copy-Item -Path (Join-Path $unzipDir $Framework "*") -Destination $Path -Recurse +Copy-Item -Path (Join-Path $unzipDir (Join-Path $Framework "*")) -Destination $Path -Recurse Copy-Item -Path (Join-Path $unzipDir "lib") -Destination $Path -Recurse # Prepare startup scripts @@ -108,11 +109,16 @@ if ($Framework -notlike "net4*") { #!/usr/bin/env pwsh dotnet (Join-Path $PSScriptRoot ipy.dll) @args '@ + if ($PSVersionTable.PSEdition -eq "Desktop" -or $IsWindows) { + $ipyPath = Join-Path $Path "ipy.bat" + Set-Content -Path $ipyPath -Value @' +@dotnet "%~dp0ipy.dll" %* +'@ + } if ($IsMacOS -or $IsLinux) { chmod +x $ipyPath chmod +x (Join-Path $Path "ipy.sh") Move-Item -Path (Join-Path $Path "ipy.sh") -Destination (Join-Path $Path "ipy") - Remove-Item -Path (Join-Path $Path "ipy.bat") } } elseif ($IsMacOS -or $IsLinux) { # Mono $ipyPath = Join-Path $Path "ipy" diff --git a/Src/Scripts/generate.py b/Src/Scripts/generate.py index c0e2979df..6b7ac5b2f 100755 --- a/Src/Scripts/generate.py +++ b/Src/Scripts/generate.py @@ -2,6 +2,7 @@ # The .NET Foundation licenses this file to you under the Apache 2.0 License. # See the LICENSE file in the project root for more information. +import functools import re import sys import os @@ -23,7 +24,8 @@ def get_root_dir(): os.path.join(root_dir, "Src", "StdLib"), ] -START = "#region Generated %s" +START_COMMON = "#region Generated" +START = START_COMMON + " %s" END = "#endregion" PREFIX = r"^([ \t]*)" @@ -131,6 +133,27 @@ def text(self): def conditions(self): return ConditionWriter(self) +@functools.lru_cache() +def find_candidates(dirname): + def listdir(dirname): + if dirname in exclude_directories: + return + + for file in os.listdir(dirname): + if file == "obj": continue # obj folders are not interesting... + filename = os.path.join(dirname, file) + if os.path.isdir(filename): + yield from listdir(filename) + elif filename.endswith(".cs") and not file == "StandardTestStrings.cs": # TODO: fix encoding of StandardTestStrings.cs + yield filename + + res = [] + for file in listdir(dirname): + with open(file, encoding='latin-1') as f: + if START_COMMON in f.read(): + res.append(file) + return res + class CodeGenerator: def __init__(self, name, generator): self.generator = generator @@ -151,19 +174,10 @@ def do_generate(self): result.append(g.generate()) return result - def do_dir(self, dirname): - if dirname in exclude_directories: - return - for file in os.listdir(dirname): - filename = os.path.join(dirname, file) - if os.path.isdir(filename): - self.do_dir(filename) - elif filename.endswith(".cs") and not file == "StandardTestStrings.cs": # TODO: fix encoding of StandardTestStrings.cs - self.do_file(filename) - def doit(self): for src_dir in source_directories: - self.do_dir(src_dir) + for file in find_candidates(src_dir): + self.do_file(file) for g in self.generators: g.collect_info() return self.do_generate() diff --git a/Tests/test_datetime.py b/Tests/test_datetime.py index 9faca7fe6..c23225566 100644 --- a/Tests/test_datetime.py +++ b/Tests/test_datetime.py @@ -126,6 +126,11 @@ def test_fromtimestamp(self): ts = 5399410716.777882 self.assertEqual(datetime.datetime.fromtimestamp(ts).microsecond, 777882) + def test_timestamp(self): + # https://github.com/IronLanguages/ironpython3/pull/1740 + dt = datetime.datetime(2000, 1, 1) + self.assertEqual(datetime.datetime.fromtimestamp(dt.timestamp()), dt) + @skipUnlessIronPython() def test_System_DateTime_conversion(self): import clr diff --git a/Tests/test_sqlite3_stdlib.py b/Tests/test_sqlite3_stdlib.py index 9f7b70356..84fcbc2c7 100644 --- a/Tests/test_sqlite3_stdlib.py +++ b/Tests/test_sqlite3_stdlib.py @@ -6,11 +6,7 @@ ## Run selected tests from sqlite3.test from StdLib ## -from iptest import is_ironpython, generate_suite, run_test, is_linux, is_netcoreapp21 - -if is_netcoreapp21: raise SystemExit # no IronPython.SQLite.dll with .NET Core 2.1 - -import test.test_sqlite +from iptest import is_ironpython, generate_suite, run_test, is_linux import sqlite3.test.dbapi import sqlite3.test.dump @@ -21,6 +17,8 @@ import sqlite3.test.types import sqlite3.test.userfunctions +import test.test_sqlite + def load_tests(loader, standard_tests, pattern): tests = loader.loadTestsFromModule(test.test_sqlite, pattern=pattern) diff --git a/Tests/test_time_stdlib.py b/Tests/test_time_stdlib.py index e3791ba4e..96357d893 100644 --- a/Tests/test_time_stdlib.py +++ b/Tests/test_time_stdlib.py @@ -6,7 +6,7 @@ ## Run selected tests from test_time from StdLib ## -from iptest import is_ironpython, generate_suite, run_test +from iptest import is_ironpython, is_osx, is_netcoreapp21, generate_suite, run_test import test.test_time @@ -14,22 +14,22 @@ def load_tests(loader, standard_tests, pattern): tests = loader.loadTestsFromModule(test.test_time) if is_ironpython: - failing_tests = [ test.test_time.TestAsctime4dyear('test_large_year'), # ValueError: year is too high test.test_time.TestAsctime4dyear('test_negative'), # ValueError: year is too low test.test_time.TimeTestCase('test_asctime'), # ValueError: year is too high test.test_time.TimeTestCase('test_asctime_bounding_check'), # ValueError: Hour, Minute, and Second parameters describe an un-representable DateTime. - test.test_time.TimeTestCase('test_clock'), # NotImplementedError: get_clock_info('clock') - test.test_time.TimeTestCase('test_get_clock_info'), # NotImplementedError: get_clock_info test.test_time.TimeTestCase('test_insane_timestamps'), # ValueError: unreasonable date/time test.test_time.TimeTestCase('test_mktime_error'), # ValueError: year is too low - test.test_time.TimeTestCase('test_process_time'), # AttributeError: 'module' object has no attribute 'process_time' test.test_time.TimeTestCase('test_strftime_bounding_check'), # ValueError: Hour, Minute, and Second parameters describe an un-representable DateTime. - test.test_time.TimeTestCase('test_time'), # NotImplementedError: get_clock_info('time') test.test_time.TimeTestCase('test_default_values_for_zero'), # AssertionError: '2000 01 01 00 00 00 1 001' != '2000 01 01 00 00 00 6 001' ] + if is_netcoreapp21 and is_osx: + failing_tests += [ + test.test_time.TimeTestCase('test_process_time'), # AssertionError + ] + skip_tests = [] return generate_suite(tests, failing_tests, skip_tests) diff --git a/WhatsNewInPython33.md b/WhatsNewInPython33.md index 7a76f99eb..08aee7fe7 100644 --- a/WhatsNewInPython33.md +++ b/WhatsNewInPython33.md @@ -117,7 +117,7 @@ Deprecated Python modules, functions and methods - [ ] `platform.popen()`: use the `subprocess` module. Check especially the "Replacing Older Functions with the `subprocess` Module" section (issue 11377). - [ ] The Windows `bytes` API has been deprecated in the `os` module. Use Unicode filenames, instead of `bytes` filenames, to not depend on the ANSI code page anymore and to support any filename. - [x] The `xml.etree.cElementTree` module is deprecated. The accelerator is used automatically whenever available. -- [ ] The behaviour of `time.clock()` depends on the platform: use the new `time.perf_counter()` or `time.process_time()` function instead, depending on your requirements, to have a well defined behaviour. +- [x] The behaviour of `time.clock()` depends on the platform: use the new `time.perf_counter()` or `time.process_time()` function instead, depending on your requirements, to have a well defined behaviour. - [x] ~~The `os.stat_float_times()` function is deprecated.~~ (Never implemented, but removed in Python 3.7) - `abc` module: + [ ] `abc.abstractproperty` has been deprecated, use `property` with `abc.abstractmethod()` instead. diff --git a/make.ps1 b/make.ps1 index 2f9867145..f14921289 100755 --- a/make.ps1 +++ b/make.ps1 @@ -4,7 +4,7 @@ Param( [Parameter(Position=1)] [String] $target = "build", [String] $configuration = "Release", - [String[]] $frameworks=@('net462','netcoreapp2.1','netcoreapp3.1','net6.0'), + [String[]] $frameworks=@('net462','netcoreapp3.1','net6.0'), [String] $platform = "x64", [switch] $runIgnored, [int] $jobs = [System.Environment]::ProcessorCount