From e54a976ebc6da742dfcd8623612e578edb0dcf40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Sun, 17 Apr 2022 15:45:07 -0400 Subject: [PATCH 1/3] Update from_buffer_copy to accept an IBufferProtocol --- Src/IronPython.Modules/_ctypes/ArrayType.cs | 35 +++++-------------- Src/IronPython.Modules/_ctypes/SimpleType.cs | 15 ++++---- Src/IronPython.Modules/_ctypes/StructType.cs | 13 ++++--- Src/IronPython.Modules/_ctypes/_ctypes.cs | 18 +--------- .../Cases/CPythonCasesManifest.ini | 2 +- Tests/test_ctypes_frombuffer_stdlib.py | 30 ++++++++++++++++ 6 files changed, 57 insertions(+), 56 deletions(-) create mode 100644 Tests/test_ctypes_frombuffer_stdlib.py diff --git a/Src/IronPython.Modules/_ctypes/ArrayType.cs b/Src/IronPython.Modules/_ctypes/ArrayType.cs index e642319a8..6fa6c0820 100644 --- a/Src/IronPython.Modules/_ctypes/ArrayType.cs +++ b/Src/IronPython.Modules/_ctypes/ArrayType.cs @@ -109,35 +109,16 @@ public _Array from_buffer(CodeContext/*!*/ context, ArrayModule.array array, [De return res; } - public _Array from_buffer_copy(CodeContext/*!*/ context, ArrayModule.array array, [DefaultParameterValue(0)] int offset) { - ValidateArraySizes(array, offset, ((INativeType)this).Size); - - _Array res = (_Array)CreateInstance(context); - res.MemHolder = new MemoryHolder(((INativeType)this).Size); - res.MemHolder.CopyFrom(array.GetArrayAddress().Add(offset), new IntPtr(((INativeType)this).Size)); - GC.KeepAlive(array); - return res; - } - - public _Array from_buffer_copy(CodeContext/*!*/ context, Bytes array, [DefaultParameterValue(0)] int offset) { - ValidateArraySizes(array, offset, ((INativeType)this).Size); + public _Array from_buffer_copy(CodeContext/*!*/ context, IBufferProtocol data, int offset = 0) { + using var buffer = data.GetBuffer(); + var span = buffer.AsReadOnlySpan(); + var size = ((INativeType)this).Size; + ValidateArraySizes(span.Length, offset, size); + span = span.Slice(offset, size); _Array res = (_Array)CreateInstance(context); - res.MemHolder = new MemoryHolder(((INativeType)this).Size); - for (int i = 0; i < ((INativeType)this).Size; i++) { - res.MemHolder.WriteByte(i, ((IList)array)[i]); - } - return res; - } - - public _Array from_buffer_copy(CodeContext/*!*/ context, string data, int offset = 0) { - ValidateArraySizes(data, offset, ((INativeType)this).Size); - - _Array res = (_Array)CreateInstance(context); - res.MemHolder = new MemoryHolder(((INativeType)this).Size); - for (int i = 0; i < ((INativeType)this).Size; i++) { - res.MemHolder.WriteByte(i, (byte)data[i]); - } + res.MemHolder = new MemoryHolder(size); + res.MemHolder.WriteSpan(0, span); return res; } diff --git a/Src/IronPython.Modules/_ctypes/SimpleType.cs b/Src/IronPython.Modules/_ctypes/SimpleType.cs index e9fe341e2..0c6bab5af 100644 --- a/Src/IronPython.Modules/_ctypes/SimpleType.cs +++ b/Src/IronPython.Modules/_ctypes/SimpleType.cs @@ -141,13 +141,16 @@ public SimpleCData from_buffer(ArrayModule.array array, int offset = 0) { return res; } - public SimpleCData from_buffer_copy(ArrayModule.array array, int offset = 0) { - ValidateArraySizes(array, offset, ((INativeType)this).Size); + public SimpleCData from_buffer_copy(CodeContext/*!*/ context, IBufferProtocol data, int offset = 0) { + using var buffer = data.GetBuffer(); + var span = buffer.AsReadOnlySpan(); + var size = ((INativeType)this).Size; + ValidateArraySizes(span.Length, offset, size); + span = span.Slice(offset, size); - SimpleCData res = (SimpleCData)CreateInstance(Context.SharedContext); - res.MemHolder = new MemoryHolder(((INativeType)this).Size); - res.MemHolder.CopyFrom(array.GetArrayAddress().Add(offset), new IntPtr(((INativeType)this).Size)); - GC.KeepAlive(array); + SimpleCData res = (SimpleCData)CreateInstance(context); + res.MemHolder = new MemoryHolder(size); + res.MemHolder.WriteSpan(0, span); return res; } diff --git a/Src/IronPython.Modules/_ctypes/StructType.cs b/Src/IronPython.Modules/_ctypes/StructType.cs index 59a032250..9141c5940 100644 --- a/Src/IronPython.Modules/_ctypes/StructType.cs +++ b/Src/IronPython.Modules/_ctypes/StructType.cs @@ -99,13 +99,16 @@ public _Structure from_buffer(CodeContext/*!*/ context, ArrayModule.array array, return res; } - public _Structure from_buffer_copy(CodeContext/*!*/ context, ArrayModule.array array, int offset = 0) { - ValidateArraySizes(array, offset, ((INativeType)this).Size); + public _Structure from_buffer_copy(CodeContext/*!*/ context, IBufferProtocol data, int offset = 0) { + using var buffer = data.GetBuffer(); + var span = buffer.AsReadOnlySpan(); + var size = ((INativeType)this).Size; + ValidateArraySizes(span.Length, offset, size); + span = span.Slice(offset, size); _Structure res = (_Structure)CreateInstance(Context.SharedContext); - res.MemHolder = new MemoryHolder(((INativeType)this).Size); - res.MemHolder.CopyFrom(array.GetArrayAddress().Add(offset), new IntPtr(((INativeType)this).Size)); - GC.KeepAlive(array); + res.MemHolder = new MemoryHolder(size); + res.MemHolder.WriteSpan(0, span); return res; } diff --git a/Src/IronPython.Modules/_ctypes/_ctypes.cs b/Src/IronPython.Modules/_ctypes/_ctypes.cs index 03266320e..60a1c8bf2 100644 --- a/Src/IronPython.Modules/_ctypes/_ctypes.cs +++ b/Src/IronPython.Modules/_ctypes/_ctypes.cs @@ -683,27 +683,11 @@ private static void ValidateArraySizes(ArrayModule.array array, int offset, int ValidateArraySizes(array.__len__() * array.itemsize, offset, size); } - private static void ValidateArraySizes(Bytes bytes, int offset, int size) { - ValidateArraySizes(bytes.Count, offset, size); - } - - private static void ValidateArraySizes(string data, int offset, int size) { - ValidateArraySizes(data.Length, offset, size); - } - private static void ValidateArraySizes(int arraySize, int offset, int size) { if (offset < 0) { throw PythonOps.ValueError("offset cannot be negative"); } else if (arraySize < size + offset) { - throw PythonOps.ValueError($"Buffer size too small ({arraySize} instead of at least {size} bytes)"); - } - } - - private static void ValidateArraySizes(BigInteger arraySize, int offset, int size) { - if (offset < 0) { - throw PythonOps.ValueError("offset cannot be negative"); - } else if (arraySize < size + offset) { - throw PythonOps.ValueError($"Buffer size too small ({arraySize} instead of at least {size} bytes)"); + throw PythonOps.ValueError($"Buffer size too small ({arraySize} instead of at least {offset + size} bytes)"); } } diff --git a/Src/IronPythonTest/Cases/CPythonCasesManifest.ini b/Src/IronPythonTest/Cases/CPythonCasesManifest.ini index ebb73068d..a1f37a73d 100644 --- a/Src/IronPythonTest/Cases/CPythonCasesManifest.ini +++ b/Src/IronPythonTest/Cases/CPythonCasesManifest.ini @@ -17,7 +17,7 @@ Ignore=true # blocked by https://github.com/IronLanguages/ironpython3/issues/129 Ignore=true Reason=Current implementation of get_last_error needs to be debugged -[CPython.ctypes.test_frombuffer] +[CPython.ctypes.test_frombuffer] # IronPython.test_ctypes_frombuffer_stdlib Ignore=true # blocked by https://github.com/IronLanguages/ironpython3/issues/1297 [CPython.ctypes.test_functions] diff --git a/Tests/test_ctypes_frombuffer_stdlib.py b/Tests/test_ctypes_frombuffer_stdlib.py new file mode 100644 index 000000000..d582303ff --- /dev/null +++ b/Tests/test_ctypes_frombuffer_stdlib.py @@ -0,0 +1,30 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the Apache 2.0 License. +# See the LICENSE file in the project root for more information. + +## +## Run selected tests from ctypes.test.test_frombuffer from StdLib +## + +import unittest +import sys + +from iptest import run_test + +import ctypes.test.test_frombuffer + +def load_tests(loader, standard_tests, pattern): + if sys.implementation.name == 'ironpython': + suite = unittest.TestSuite() + suite.addTest(ctypes.test.test_frombuffer.Test('test_fortran_contiguous')) + suite.addTest(unittest.expectedFailure(ctypes.test.test_frombuffer.Test('test_from_buffer'))) + suite.addTest(ctypes.test.test_frombuffer.Test('test_from_buffer_copy')) + suite.addTest(ctypes.test.test_frombuffer.Test('test_from_buffer_copy_with_offset')) + suite.addTest(unittest.expectedFailure(ctypes.test.test_frombuffer.Test('test_from_buffer_memoryview'))) + suite.addTest(ctypes.test.test_frombuffer.Test('test_from_buffer_with_offset')) + return suite + + else: + return loader.loadTestsFromModule(ctypes.test.test_frombuffer, pattern) + +run_test(__name__) From 5447f34df43dc95239c9fffd2054419aabbdab9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Mon, 18 Apr 2022 20:26:26 -0400 Subject: [PATCH 2/3] Update after review --- Src/IronPython.Modules/_ctypes/ArrayType.cs | 3 ++- Src/IronPython.Modules/_ctypes/MemoryHolder.cs | 8 +++++--- Src/IronPython.Modules/_ctypes/SimpleType.cs | 2 +- Src/IronPython.Modules/_ctypes/StructType.cs | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Src/IronPython.Modules/_ctypes/ArrayType.cs b/Src/IronPython.Modules/_ctypes/ArrayType.cs index 6fa6c0820..8715e3f79 100644 --- a/Src/IronPython.Modules/_ctypes/ArrayType.cs +++ b/Src/IronPython.Modules/_ctypes/ArrayType.cs @@ -15,6 +15,7 @@ using IronPython.Runtime.Operations; using IronPython.Runtime.Types; +using Microsoft.Scripting.Runtime; using Microsoft.Scripting.Utils; namespace IronPython.Modules { @@ -109,7 +110,7 @@ public _Array from_buffer(CodeContext/*!*/ context, ArrayModule.array array, [De return res; } - public _Array from_buffer_copy(CodeContext/*!*/ context, IBufferProtocol data, int offset = 0) { + public _Array from_buffer_copy(CodeContext/*!*/ context, [NotNull] IBufferProtocol data, int offset = 0) { using var buffer = data.GetBuffer(); var span = buffer.AsReadOnlySpan(); var size = ((INativeType)this).Size; diff --git a/Src/IronPython.Modules/_ctypes/MemoryHolder.cs b/Src/IronPython.Modules/_ctypes/MemoryHolder.cs index 82028bd84..1cdc4c846 100644 --- a/Src/IronPython.Modules/_ctypes/MemoryHolder.cs +++ b/Src/IronPython.Modules/_ctypes/MemoryHolder.cs @@ -10,7 +10,6 @@ using System.Runtime.CompilerServices; using System.Runtime.ConstrainedExecution; using System.Runtime.InteropServices; -using System.Text; using System.Threading; using IronPython.Runtime; @@ -282,10 +281,13 @@ internal void WriteUnicodeString(int offset, string value) { WriteInt16(checked(offset + i * 2), (short)value[i]); } } + internal void WriteSpan(int offset, ReadOnlySpan value) { - for (int i = 0; i < value.Length; i++) { - WriteByte(checked(offset + i), value[i]); + unsafe { + var dest = new Span((void*)_data, _size).Slice(offset); + value.CopyTo(dest); } + GC.KeepAlive(this); } public MemoryHolder GetSubBlock(int offset) { diff --git a/Src/IronPython.Modules/_ctypes/SimpleType.cs b/Src/IronPython.Modules/_ctypes/SimpleType.cs index 0c6bab5af..cc9a3f779 100644 --- a/Src/IronPython.Modules/_ctypes/SimpleType.cs +++ b/Src/IronPython.Modules/_ctypes/SimpleType.cs @@ -141,7 +141,7 @@ public SimpleCData from_buffer(ArrayModule.array array, int offset = 0) { return res; } - public SimpleCData from_buffer_copy(CodeContext/*!*/ context, IBufferProtocol data, int offset = 0) { + public SimpleCData from_buffer_copy(CodeContext/*!*/ context, [NotNull] IBufferProtocol data, int offset = 0) { using var buffer = data.GetBuffer(); var span = buffer.AsReadOnlySpan(); var size = ((INativeType)this).Size; diff --git a/Src/IronPython.Modules/_ctypes/StructType.cs b/Src/IronPython.Modules/_ctypes/StructType.cs index 9141c5940..30c173ebb 100644 --- a/Src/IronPython.Modules/_ctypes/StructType.cs +++ b/Src/IronPython.Modules/_ctypes/StructType.cs @@ -99,14 +99,14 @@ public _Structure from_buffer(CodeContext/*!*/ context, ArrayModule.array array, return res; } - public _Structure from_buffer_copy(CodeContext/*!*/ context, IBufferProtocol data, int offset = 0) { + public _Structure from_buffer_copy(CodeContext/*!*/ context, [NotNull] IBufferProtocol data, int offset = 0) { using var buffer = data.GetBuffer(); var span = buffer.AsReadOnlySpan(); var size = ((INativeType)this).Size; ValidateArraySizes(span.Length, offset, size); span = span.Slice(offset, size); - _Structure res = (_Structure)CreateInstance(Context.SharedContext); + _Structure res = (_Structure)CreateInstance(context); res.MemHolder = new MemoryHolder(size); res.MemHolder.WriteSpan(0, span); return res; From 101bea43edef33fd71b0b3cbbda3d2f00ef65945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Mon, 18 Apr 2022 21:11:49 -0400 Subject: [PATCH 3/3] Fix _pack_ --- Src/IronPython.Modules/_ctypes/StructType.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Src/IronPython.Modules/_ctypes/StructType.cs b/Src/IronPython.Modules/_ctypes/StructType.cs index 30c173ebb..b015158c8 100644 --- a/Src/IronPython.Modules/_ctypes/StructType.cs +++ b/Src/IronPython.Modules/_ctypes/StructType.cs @@ -50,10 +50,15 @@ public StructType(CodeContext/*!*/ context, string name, PythonTuple bases, Pyth } if (members.TryGetValue("_pack_", out object pack)) { - if (!(pack is int) || ((int)pack < 0)) { + object index = PythonOps.Index(pack); // since 3.8 + _pack = index switch { + int i => i, + BigInteger bi => (int)bi, // CPython throws the ValueError below on overflow + _ => throw new InvalidOperationException(), + }; + if (_pack < 0) { throw PythonOps.ValueError("pack must be a non-negative integer"); } - _pack = (int)pack; } if (members.TryGetValue("_fields_", out object fields)) {