From dea0bfef2de2be7f2f47a0f510297fb429c83646 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Fri, 25 Sep 2020 12:25:55 -0700 Subject: [PATCH 1/4] Use __index__ operator on more code paths --- Src/IronPython/Runtime/ByteArray.cs | 26 ++++++++++++++------------ Src/IronPython/Runtime/Bytes.cs | 3 +++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Src/IronPython/Runtime/ByteArray.cs b/Src/IronPython/Runtime/ByteArray.cs index ebc5516d0..a21a64176 100644 --- a/Src/IronPython/Runtime/ByteArray.cs +++ b/Src/IronPython/Runtime/ByteArray.cs @@ -68,24 +68,26 @@ public void __init__(int source) { } } - public void __init__([NotNull]IEnumerable source) { - lock (this) { - _bytes.Clear(); - _bytes.AddRange(source); - } - } - - public void __init__([BytesLike, NotNull]IBufferProtocol source) { - lock (this) { - _bytes.Clear(); - using IPythonBuffer buffer = source.GetBuffer(BufferFlags.FullRO); - _bytes.AddRange(buffer); + public void __init__([NotNull]IBufferProtocol source) { + if (Converter.TryConvertToIndex(source, throwOverflowError: true, out int size)) { + __init__(size); + } else { + lock (this) { + _bytes.Clear(); + using IPythonBuffer buffer = source.GetBuffer(BufferFlags.FullRO); + _bytes.AddRange(buffer); + } } } public void __init__(CodeContext context, object? source) { if (Converter.TryConvertToIndex(source, throwOverflowError: true, out int size)) { __init__(size); + } else if (source is IEnumerable en) { + lock (this) { + _bytes.Clear(); + _bytes.AddRange(en); + } } else { lock (this) { _bytes.Clear(); diff --git a/Src/IronPython/Runtime/Bytes.cs b/Src/IronPython/Runtime/Bytes.cs index 49d28b22d..fbe3538d5 100644 --- a/Src/IronPython/Runtime/Bytes.cs +++ b/Src/IronPython/Runtime/Bytes.cs @@ -59,6 +59,9 @@ public static object __new__(CodeContext context, [NotNull] PythonType cls, [Not return source; } else if (TryInvokeBytesOperator(context, source, out Bytes? res)) { return res; + } else if (Converter.TryConvertToIndex(source, throwOverflowError: true, out int size)) { + if (size < 0) throw PythonOps.ValueError("negative count"); + return new Bytes(new byte[size]); } else { return new Bytes(source); } From 3828f679d270206e5a58d078a84911a90bfbf7b1 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Fri, 25 Sep 2020 12:27:57 -0700 Subject: [PATCH 2/4] Test __index__ use in construction of bytes, bytearray --- Tests/test_bytes.py | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Tests/test_bytes.py b/Tests/test_bytes.py index 46c00f427..3a7bbc49f 100644 --- a/Tests/test_bytes.py +++ b/Tests/test_bytes.py @@ -244,6 +244,52 @@ def __bytes__(self): self.assertEqual(BytesSubclass(IntSubclass(-1)), b"BYTES FROM INT") self.assertIs(type(BytesSubclass(IntSubclass(-1))), BytesSubclass) + def test_dunder_index(self): + class IndexableBytes(bytes): + def __init__(self, value): + self.value = len(value) + def __index__(self): + return self.value + + class IndexableBytearray(bytearray): + def __init__(self, value): + super().__init__(value) + self.value = len(value) + def __index__(self): + return self.value + + class IndexableStr(str): + def __init__(self, value): + self.value = len(value) + def __index__(self): + return self.value + + class IndexableInt(int): + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + 10 + + ib = IndexableBytes(b"xyz") + iba = IndexableBytearray(b"abcd") + istr = IndexableStr("abcde") + ii = IndexableInt(2) + self.assertEqual(ii.__index__(), 12) + + self.assertEqual(ib, b"xyz") + self.assertEqual(iba, bytearray(b"abcd")) + self.assertEqual(istr, "abcde") + + self.assertEqual(bytes(ib), bytes(3)) + self.assertEqual(bytes(iba), bytes(4)) + self.assertRaises(TypeError, bytes, istr) + self.assertEqual(bytes(ii), bytes(2)) + + self.assertEqual(bytearray(ib), bytearray(3)) + self.assertEqual(bytearray(iba), bytearray(4)) + self.assertRaises(TypeError, bytearray, istr) + self.assertEqual(bytearray(ii), bytes(2)) + def test_capitalize(self): tests = [(b'foo', b'Foo'), (b' foo', b' foo'), From 92f4b19666266ec5b5e05a606e576538957aeb67 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Fri, 25 Sep 2020 12:50:58 -0700 Subject: [PATCH 3/4] Remove support for BytesLikeAttribute on IBufferProtocol --- .../IronPythonAnalyzerAnalyzer.cs | 4 +--- .../Runtime/Binding/PythonOverloadResolver.cs | 16 ---------------- Src/IronPython/Runtime/BytesLikeAttribute.cs | 7 ++----- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/IronPythonAnalyzer/IronPythonAnalyzer/IronPythonAnalyzerAnalyzer.cs b/IronPythonAnalyzer/IronPythonAnalyzer/IronPythonAnalyzerAnalyzer.cs index c879361b6..90e96c991 100644 --- a/IronPythonAnalyzer/IronPythonAnalyzer/IronPythonAnalyzerAnalyzer.cs +++ b/IronPythonAnalyzer/IronPythonAnalyzer/IronPythonAnalyzerAnalyzer.cs @@ -19,7 +19,7 @@ public class IronPythonAnalyzerAnalyzer : DiagnosticAnalyzer { private static readonly DiagnosticDescriptor Rule1 = new DiagnosticDescriptor("IPY01", title: "Parameter which is marked not nullable does not have the NotNullAttribute", messageFormat: "Parameter '{0}' does not have the NotNullAttribute", category: "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: "Non-nullable reference type parameters should have the NotNullAttribute."); private static readonly DiagnosticDescriptor Rule2 = new DiagnosticDescriptor("IPY02", title: "Parameter which is marked nullable has the NotNullAttribute", messageFormat: "Parameter '{0}' should not have the NotNullAttribute", category: "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: "Nullable reference type parameters should not have the NotNullAttribute."); - private static readonly DiagnosticDescriptor Rule3 = new DiagnosticDescriptor("IPY03", title: "BytesLikeAttribute used on a not supported type", messageFormat: "Parameter '{0}' declared bytes-like on unsupported type '{1}'", category: "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: "BytesLikeAttribute is only allowed on parameters of type ReadOnlyMemory, IReadOnlyList, or IList."); + private static readonly DiagnosticDescriptor Rule3 = new DiagnosticDescriptor("IPY03", title: "BytesLikeAttribute used on a not supported type", messageFormat: "Parameter '{0}' declared bytes-like on unsupported type '{1}'", category: "Usage", DiagnosticSeverity.Warning, isEnabledByDefault: true, description: "BytesLikeAttribute is only allowed on parameters of type IReadOnlyList, or IList."); public override ImmutableArray SupportedDiagnostics { get { return ImmutableArray.Create(Rule1, Rule2, Rule3); } } @@ -57,11 +57,9 @@ private static void AnalyzeSymbol(SymbolAnalysisContext context) { var ilistType = context.Compilation.GetTypeByMetadataName("System.Collections.Generic.IList`1"); var ireadOnlyListOfByteType = ireadOnlyListType.Construct(byteType); var ilistOfByteType = ilistType.Construct(byteType); - var ibufferProtocolType = context.Compilation.GetTypeByMetadataName("IronPython.Runtime.IBufferProtocol"); foreach (IParameterSymbol parameterSymbol in methodSymbol.Parameters) { if (parameterSymbol.GetAttributes().Any(x => x.AttributeClass.Equals(bytesLikeAttributeSymbol)) - && !parameterSymbol.Type.Equals(ibufferProtocolType) && !parameterSymbol.Type.Equals(ireadOnlyListOfByteType) && !parameterSymbol.Type.Equals(ilistOfByteType)) { var diagnostic = Diagnostic.Create(Rule3, parameterSymbol.Locations[0], parameterSymbol.Name, parameterSymbol.Type.MetadataName); diff --git a/Src/IronPython/Runtime/Binding/PythonOverloadResolver.cs b/Src/IronPython/Runtime/Binding/PythonOverloadResolver.cs index e915e2895..fc78db46b 100644 --- a/Src/IronPython/Runtime/Binding/PythonOverloadResolver.cs +++ b/Src/IronPython/Runtime/Binding/PythonOverloadResolver.cs @@ -91,22 +91,6 @@ public override Candidate SelectBestConversionFor(DynamicMetaObject arg, Paramet } } - bool isBytesLikeOne = IsBytesLikeParameter(candidateOne); - bool isBytesLikeTwo = IsBytesLikeParameter(candidateTwo); - - if (isBytesLikeOne) { - if (isBytesLikeTwo) { - return basePreferred; - } - if (candidateTwo.Type.IsInterface) { - return Candidate.One; - } - } else if (isBytesLikeTwo) { - if (candidateOne.Type.IsInterface) { - return Candidate.Two; - } - } - return basePreferred; } diff --git a/Src/IronPython/Runtime/BytesLikeAttribute.cs b/Src/IronPython/Runtime/BytesLikeAttribute.cs index a5732fa00..69a88440a 100644 --- a/Src/IronPython/Runtime/BytesLikeAttribute.cs +++ b/Src/IronPython/Runtime/BytesLikeAttribute.cs @@ -8,14 +8,11 @@ namespace IronPython.Runtime { /// - /// For IList〈byte〉, IReadOnlyList〈byte〉, and IBufferProtocol parameters: + /// For IList〈byte〉 and IReadOnlyList〈byte〉 parameters: /// Marks that the parameter is typed to accept a bytes-like object. ///
- /// If applied on a IList〈byte〉 parameter, this attribute disallows passing + /// It also disallows passing /// a Python list object and auto-applying our generic conversion. - ///
- /// The overload resolver will favor an overload with a BytesLike parameter - /// over an otherwise equivalent overload with a different interface parameter. /// /// /// A bytes-like object is any object of type implementing IBufferProtocol. From 1876b6e1602b179de7c6eecaca4570a68a9e951c Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Fri, 25 Sep 2020 15:28:50 -0700 Subject: [PATCH 4/4] Sanitize Bytes.FromObject --- Src/IronPython/Runtime/Bytes.cs | 14 ++++++++++---- Tests/test_bytes.py | 9 +++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Src/IronPython/Runtime/Bytes.cs b/Src/IronPython/Runtime/Bytes.cs index fbe3538d5..0592d7061 100644 --- a/Src/IronPython/Runtime/Bytes.cs +++ b/Src/IronPython/Runtime/Bytes.cs @@ -73,7 +73,16 @@ public static object __new__(CodeContext context, [NotNull] PythonType cls, [Not [StaticExtensionMethod] public static object __new__(CodeContext context, [NotNull] PythonType cls, object? @object) { if (cls == TypeCache.Bytes) { - return FromObject(context, @object); + if (@object?.GetType() == typeof(Bytes)) { + return @object; + } else if (TryInvokeBytesOperator(context, @object, out Bytes? res)) { + return res; + } else if (Converter.TryConvertToIndex(@object, throwOverflowError: true, out int size)) { + if (size < 0) throw PythonOps.ValueError("negative count"); + return new Bytes(new byte[size]); + } else { + return new Bytes(ByteOps.GetBytes(@object, useHint: true, context).ToArray()); + } } else { return cls.CreateInstance(context, __new__(context, TypeCache.Bytes, @object)); } @@ -155,9 +164,6 @@ internal static Bytes FromObject(CodeContext context, object? o) { return (Bytes)o; } else if (TryInvokeBytesOperator(context, o, out Bytes? res)) { return res; - } else if (Converter.TryConvertToIndex(o, throwOverflowError: true, out int size)) { - if (size < 0) throw PythonOps.ValueError("negative count"); - return new Bytes(new byte[size]); } else { return new Bytes(ByteOps.GetBytes(o, useHint: true, context).ToArray()); } diff --git a/Tests/test_bytes.py b/Tests/test_bytes.py index 3a7bbc49f..35fa5b156 100644 --- a/Tests/test_bytes.py +++ b/Tests/test_bytes.py @@ -203,6 +203,7 @@ def __bytes__(self): self.assertIs(type(bytes(SomeClass())), OtherBytesSubclass) self.assertEqual(BytesSubclass(SomeClass()), b'SOME CLASS') self.assertIs(type(BytesSubclass(SomeClass())), BytesSubclass) + self.assertEqual(int.from_bytes(bytes(SomeClass()), 'big'), int.from_bytes(b"SOME CLASS", 'big')) class BytesBytesSubclass(bytes): def __bytes__(self): @@ -210,6 +211,7 @@ def __bytes__(self): self.assertEqual(bytes(BytesBytesSubclass(b"JUST BYTES")), b"BYTES FROM BYTES") self.assertIs(type(bytes(BytesBytesSubclass(b"JUST BYTES"))), BytesBytesSubclass) + self.assertEqual(int.from_bytes(bytes(BytesBytesSubclass(b"JUST BYTES")), 'big'), int.from_bytes(b"BYTES FROM BYTES", 'big')) class ListSubclass(bytes): def __bytes__(self): @@ -243,6 +245,7 @@ def __bytes__(self): self.assertIs(type(bytes(IntSubclass(-1))), OtherBytesSubclass) self.assertEqual(BytesSubclass(IntSubclass(-1)), b"BYTES FROM INT") self.assertIs(type(BytesSubclass(IntSubclass(-1))), BytesSubclass) + self.assertEqual(int.from_bytes(IntSubclass(555), 'big'), int.from_bytes(b"BYTES FROM INT", 'big')) def test_dunder_index(self): class IndexableBytes(bytes): @@ -290,6 +293,12 @@ def __index__(self): self.assertRaises(TypeError, bytearray, istr) self.assertEqual(bytearray(ii), bytes(2)) + self.assertEqual(int.from_bytes(IndexableBytes(b"abc"), 'big'), 0x616263) + self.assertEqual(int.from_bytes(IndexableBytearray(b"abc"), 'big'), 0x616263) + self.assertRaises(TypeError, int.from_bytes, IndexableStr("abc"), 'big') + self.assertRaises(TypeError, int.from_bytes, IndexableInt(2), 'big') + self.assertRaises(TypeError, int.from_bytes, 2, 'big') + def test_capitalize(self): tests = [(b'foo', b'Foo'), (b' foo', b' foo'),