Re-enable some tests

Src/IronPython.Modules/_collections.cs

@@ -551,20 +551,13 @@ public void __delitem__(CodeContext/*!*/ context, object index) {
                 }
             }
 
-            public PythonTuple __reduce__() {
-                lock (_lockObj) {
-                    object[] items = new object[_itemCnt];
-                    int curItem = 0;
-                    WalkDeque(delegate(int curIndex) {
-                        items[curItem++] = _data[curIndex];
-                        return true;
-                    });
-
-                    return PythonTuple.MakeTuple(
-                        DynamicHelpers.GetPythonType(this),
-                        PythonTuple.MakeTuple(PythonList.FromArrayNoCopy(items))
-                    );
-                }
+            public PythonTuple __reduce__(CodeContext context) {
+                return PythonTuple.MakeTuple(
+                    DynamicHelpers.GetPythonType(this),
+                    _maxLen == -1 ? PythonTuple.EMPTY : PythonTuple.MakeTuple(PythonTuple.EMPTY, maxlen),
+                    GetType() == typeof(deque) ? null : PythonOps.GetBoundAttr(context, this, "__dict__"),
+                    PythonOps.GetEnumeratorObject(context, this)
+                );
             }
 
             public int __len__() {

Src/IronPython.Modules/_ctypes/CData.cs

@@ -72,8 +72,7 @@ internal void SetAddress(IntPtr address) {
 
             internal void InitializeFromBuffer(object? data, int offset, int size) {
                 var bp = data as IBufferProtocol
-                    ?? throw PythonOps.TypeErrorForBadInstance("{0} object does not have the buffer interface", data);
-                    // Python 3.5: PythonOps.TypeErrorForBytesLikeTypeMismatch(data);
+                    ?? throw PythonOps.TypeErrorForBytesLikeTypeMismatch(data);
 
                 IPythonBuffer buffer = bp.GetBuffer(BufferFlags.FullRO);
                 if (buffer.IsReadOnly) {
@@ -97,8 +96,7 @@ internal void InitializeFromBuffer(object? data, int offset, int size) {
 
             internal void InitializeFromBufferCopy(object? data, int offset, int size) {
                 var bp = data as IBufferProtocol
-                    ?? throw PythonOps.TypeErrorForBadInstance("{0} object does not have the buffer interface", data);
-                    // Python 3.5: PythonOps.TypeErrorForBytesLikeTypeMismatch(data);
+                    ?? throw PythonOps.TypeErrorForBytesLikeTypeMismatch(data);
 
                 using IPythonBuffer buffer = bp.GetBuffer();
                 var span = buffer.AsReadOnlySpan();

Src/IronPython.Modules/_ctypes/CFuncPtrType.cs

@@ -69,6 +69,12 @@ internal static PythonType MakeSystemType(Type underlyingSystemType) {
                 return PythonType.SetPythonType(underlyingSystemType, new CFuncPtrType(underlyingSystemType));
             }
 
+            public object from_buffer(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
+            public object from_buffer_copy(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
             /// <summary>
             /// Converts an object into a function call parameter.
             /// </summary>

Src/IronPython.Modules/_ctypes/PointerType.cs

@@ -47,6 +47,12 @@ private PointerType(Type underlyingSystemType)
                 : base(underlyingSystemType) {
             }
 
+            public object from_buffer(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
+            public object from_buffer_copy(object obj)
+                => throw PythonOps.TypeError("abstract class");
+
             public object from_param([NotNone] CData obj) {
                 return new NativeArgument((CData)PythonCalls.Call(this, obj), "P");
             }

Src/IronPython.Modules/_ctypes/_ctypes.cs

@@ -101,27 +101,21 @@ private static IntPtr Cast(IntPtr data, IntPtr obj, IntPtr type) {
             GCHandle typeHandle = GCHandle.FromIntPtr(type);
             try {
                 CData cdata = objHandle.Target as CData;
-                PythonType pt = (PythonType)typeHandle.Target;
+                PythonType pt = typeHandle.Target as PythonType;
 
-                CData res = (CData)pt.CreateInstance(pt.Context.SharedContext);
-                if (IsPointer(pt)) {
-                    res.MemHolder = new MemoryHolder(IntPtr.Size);
-                    if (IsPointer(DynamicHelpers.GetPythonType(cdata))) {
-                        res.MemHolder.WriteIntPtr(0, cdata.MemHolder.ReadIntPtr(0));
-                    } else {
-                        res.MemHolder.WriteIntPtr(0, data);
-                    }
+                if (!IsPointer(pt)) throw PythonOps.TypeError("cast() argument 2 must be a pointer type, not {0}", PythonOps.GetPythonTypeName(typeHandle.Target));
 
-                    if (cdata != null) {
-                        res.MemHolder.Objects = cdata.MemHolder.Objects;
-                        res.MemHolder.AddObject(IdDispenser.GetId(cdata), cdata);
-                    }
+                CData res = (CData)pt.CreateInstance(pt.Context.SharedContext);
+                res.MemHolder = new MemoryHolder(IntPtr.Size);
+                if (IsPointer(DynamicHelpers.GetPythonType(cdata))) {
+                    res.MemHolder.WriteIntPtr(0, cdata.MemHolder.ReadIntPtr(0));
                 } else {
-                    if (cdata != null) {
-                        res.MemHolder = new MemoryHolder(data, ((INativeType)pt).Size, cdata.MemHolder);
-                    } else {
-                        res.MemHolder = new MemoryHolder(data, ((INativeType)pt).Size);
-                    }
+                    res.MemHolder.WriteIntPtr(0, data);
+                }
+
+                if (cdata != null) {
+                    res.MemHolder.Objects = cdata.MemHolder.Objects;
+                    res.MemHolder.AddObject(IdDispenser.GetId(cdata), cdata);
                 }
 
                 return GCHandle.ToIntPtr(GCHandle.Alloc(res));

Src/IronPython.Modules/mmap.cs

@@ -727,20 +727,23 @@ public object tell() {
                 }
             }
 
-            public void write([BytesLike] IList<byte> s) {
+            public int write([NotNone] IBufferProtocol s) {
+                using var buffer = s.GetBuffer();
                 using (new MmapLocker(this)) {
                     EnsureWritable();
 
                     long pos = Position;
 
-                    if (_view.Capacity - pos < s.Count) {
+                    if (_view.Capacity - pos < buffer.AsReadOnlySpan().Length) {
                         throw PythonOps.ValueError("data out of range");
                     }
 
-                    byte[] data = s as byte[] ?? (s is Bytes b ? b.UnsafeByteArray : s.ToArray());
-                    _view.WriteArray(pos, data, 0, s.Count);
+                    byte[] data = buffer.AsUnsafeArray() ?? buffer.ToArray();
+                    _view.WriteArray(pos, data, 0, data.Length);
 
-                    Position = pos + s.Count;
+                    Position = pos + data.Length;
+
+                    return data.Length;
                 }
             }
 

Src/IronPython/Compiler/Ast/TupleExpression.cs

@@ -17,17 +17,10 @@ public TupleExpression(bool expandable, params Expression[] items)
         }
 
         internal override string? CheckAssign() {
-            if (Items.Count == 0) {
-                //  TODO: remove this when we get to 3.6
-                return "can't assign to ()";
-            }
-
             return base.CheckAssign();
         }
 
         internal override string? CheckDelete() {
-            if (Items.Count == 0)
-                return "can't delete ()"; //  TODO: remove this when we get to 3.6
             return base.CheckDelete();
         }
 

Src/IronPython/Modules/Builtin.cs

@@ -135,7 +135,7 @@ public static string bin(object? number) {
 
         public static PythonType bytes => DynamicHelpers.GetPythonTypeFromType(typeof(Bytes));
 
-        public static PythonType bytearray => DynamicHelpers.GetPythonTypeFromType(typeof(ByteArray));
+        public static PythonType bytearray => TypeCache.ByteArray;
 
         [Documentation("callable(object) -> bool\n\nReturn whether the object is callable (i.e., some kind of function).")]
         public static bool callable(CodeContext/*!*/ context, object? o) {

Src/IronPython/Runtime/ByteArray.cs

@@ -9,16 +9,16 @@
 using System.Collections.Generic;
 using System.Numerics;
 using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
 using System.Text;
 
-using Microsoft.Scripting.Runtime;
-using Microsoft.Scripting.Utils;
-
 using IronPython.Runtime.Exceptions;
 using IronPython.Runtime.Operations;
 using IronPython.Runtime.Types;
 
+using Microsoft.Scripting;
+using Microsoft.Scripting.Runtime;
+using Microsoft.Scripting.Utils;
+
 namespace IronPython.Runtime {
     /// <summary>
     /// bytearray(string, encoding[, errors]) -> bytearray
@@ -56,6 +56,12 @@ internal ByteArray(IEnumerable<byte> bytes) {
             _bytes = new ArrayData<byte>(bytes);
         }
 
+        [StaticExtensionMethod]
+        public static object __new__(CodeContext context, [NotNone] PythonType cls, [ParamDictionary, NotNone] IDictionary<object, object> dict, [NotNone] params object[] args) {
+            if (cls == TypeCache.ByteArray) return new ByteArray();
+            return cls.CreateInstance(context);
+        }
+
         public void __init__() {
             lock (this) {
                 _bytes.Clear();
@@ -74,7 +80,7 @@ public void __init__(int source) {
         }
 
         public void __init__([NotNone] IBufferProtocol source) {
-            if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) {
+            if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) {
                 __init__(size);
             } else {
                 lock (this) {
@@ -86,7 +92,7 @@ public void __init__([NotNone] IBufferProtocol source) {
         }
 
         public void __init__(CodeContext context, object? source) {
-            if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) {
+            if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) {
                 __init__(size);
             } else if (source is IEnumerable<byte> en) {
                 lock (this) {
@@ -466,8 +472,13 @@ public int find(BigInteger @byte, object? start, object? end) {
             }
         }
 
-        public static ByteArray fromhex([NotNone] string @string) {
-            return new ByteArray(IListOfByteOps.FromHex(@string));
+        [ClassMethod]
+        public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string) {
+            var hex = IListOfByteOps.FromHex(@string);
+            if (cls == TypeCache.ByteArray) {
+                return new ByteArray(hex);
+            }
+            return PythonTypeOps.CallParams(context, cls, new Bytes(hex));
         }
 
         public string hex() => Bytes.ToHex(_bytes.AsByteSpan()); // new in CPython 3.5

Src/IronPython/Runtime/Bytes.cs

@@ -61,7 +61,7 @@ public static object __new__(CodeContext context, [NotNone] PythonType cls, [Not
                     return source;
                 } else if (TryInvokeBytesOperator(context, source, out Bytes? res)) {
                     return res;
-                } else if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) {
+                } else if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) {
                     if (size < 0) throw PythonOps.ValueError("negative count");
                     return new Bytes(new byte[size]);
                 } else {
@@ -79,7 +79,7 @@ public static object __new__(CodeContext context, [NotNone] PythonType cls, obje
                     return @object;
                 } else if (TryInvokeBytesOperator(context, @object, out Bytes? res)) {
                     return res;
-                } else if (Converter.TryConvertToIndex(@object, out int size, throwNonInt: false)) {
+                } else if (Converter.TryConvertToIndex(@object, out int size, throwTypeError: false)) {
                     if (size < 0) throw PythonOps.ValueError("negative count");
                     return new Bytes(new byte[size]);
                 } else {
@@ -367,8 +367,13 @@ public int find(BigInteger @byte, object? start, object? end) {
         }
 
         [ClassMethod]
-        public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string)
-            => __new__(context, cls, IListOfByteOps.FromHex(@string));
+        public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string) {
+            var hex = IListOfByteOps.FromHex(@string);
+            if (cls == TypeCache.Bytes) {
+                return new Bytes(hex);
+            }
+            return PythonTypeOps.CallParams(context, cls, new Bytes(hex));
+        }
 
         public string hex() => ToHex(_bytes.AsSpan()); // new in CPython 3.5
 
@@ -1181,6 +1186,13 @@ public bool __eq__(CodeContext context, [NotNone] string value) {
 
         public bool __eq__(CodeContext context, [NotNone] Extensible<string> value) => __eq__(context, value.Value);
 
+        public bool __eq__(CodeContext context, [NotNone] int value) {
+            if (context.LanguageContext.PythonOptions.BytesWarning != Microsoft.Scripting.Severity.Ignore) {
+                PythonOps.Warn(context, PythonExceptions.BytesWarning, "Comparison between bytes and int");
+            }
+            return false;
+        }
+
         [return: MaybeNotImplemented]
         public NotImplementedType __eq__(CodeContext context, object? value) => NotImplementedType.Value;
 
@@ -1190,6 +1202,8 @@ public bool __eq__(CodeContext context, [NotNone] string value) {
 
         public bool __ne__(CodeContext context, [NotNone] Extensible<string> value) => !__eq__(context, value);
 
+        public bool __ne__(CodeContext context, [NotNone] int value) => !__eq__(context, value);
+
         [return: MaybeNotImplemented]
         public NotImplementedType __ne__(CodeContext context, object? value) => NotImplementedType.Value;
 

Src/IronPython/Runtime/Converter.cs

@@ -335,28 +335,34 @@ public static IEnumerable ConvertToIEnumerable(object o) {
         /// If throwOverflowError is true then BigInteger's outside the normal range of integers will
         /// result in an OverflowError.
         ///
-        /// When throwNonInt is true, a TypeError will be thrown if __index__ returned a non-int.
+        /// When throwTypeError is true, a TypeError will be thrown if __index__ throw a TypeError or
+        /// returned a non-int.
         /// </summary>
-        internal static bool TryConvertToIndex(object? value, out int index, bool throwOverflowError = true, bool throwNonInt = true) {
+        internal static bool TryConvertToIndex(object? value, out int index, bool throwOverflowError = true, bool throwTypeError = true) {
             if (TryGetInt(value, out index, throwOverflowError, value)) {
                 return true;
             }
 
-            if (PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, value, "__index__", out object indexObj)) {
-                if (TryGetInt(indexObj, out index, throwOverflowError, value)) {
-                    return true;
-                }
+            if (throwTypeError) {
+                if (PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, value, "__index__", out object indexObj)) {
+                    if (TryGetInt(indexObj, out index, throwOverflowError, value)) {
+                        return true;
+                    }
 
-                if (throwNonInt) {
                     throw PythonOps.TypeError("__index__ returned non-int (type {0})", PythonOps.GetPythonTypeName(indexObj));
                 }
+            } else {
+                try {
+                    return PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, value, "__index__", out object indexObj) &&
+                        TryGetInt(indexObj, out index, throwOverflowError, value);
+                } catch (Exceptions.TypeErrorException) { }
             }
 
             return false;
         }
 
         public static int ConvertToIndex(object? value, bool throwOverflowError = false) {
-            if (TryConvertToIndex(value, out int index, throwOverflowError: throwOverflowError, throwNonInt: true))
+            if (TryConvertToIndex(value, out int index, throwOverflowError: throwOverflowError))
                 return index;
 
             throw PythonOps.TypeError("expected integer value, got {0}", PythonOps.GetPythonTypeName(value));

Src/IronPython/Runtime/Operations/BigIntegerOps.cs

@@ -924,10 +924,9 @@ public static object from_bytes(CodeContext context, PythonType type, object? by
             var val = new BigInteger(bytesArr);
 #endif
 
-            // prevents a TypeError: int.__new__(bool) is not safe
-            if (type == TypeCache.Boolean) return val == 0 ? ScriptingRuntimeHelpers.False : ScriptingRuntimeHelpers.True;
+            if (type == TypeCache.BigInteger) return val;
 
-            return __new__(context, type, val);
+            return PythonTypeOps.CallParams(context, type, val);
         }
 
         #endregion

Src/IronPython/Runtime/Operations/PythonOps.cs

@@ -758,7 +758,7 @@ public static object PowerMod(CodeContext/*!*/ context, object? x, object? y, ob
                 }
             }
 
-            throw PythonOps.TypeErrorForBinaryOp("power with modulus", x, y);
+            throw PythonOps.TypeError("unsupported operand type(s) for pow(): '{0}', '{1}', '{2}'", GetPythonTypeName(x), GetPythonTypeName(y), GetPythonTypeName(y));
         }
 
         public static long Id(object? o) {
@@ -896,10 +896,16 @@ public static int Length(object? o) {
         }
 
         internal static bool TryInvokeLengthHint(CodeContext context, object? sequence, out int hint) {
-            if (PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__len__", out object len_obj) ||
-                PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__length_hint__", out len_obj)) {
+            if (PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__len__", out object len_obj)) {
                 if (!(len_obj is NotImplementedType)) {
                     hint = Converter.ConvertToInt32(len_obj);
+                    if (hint < 0) throw ValueError("__len__() should return >= 0");
+                    return true;
+                }
+            } else if (PythonTypeOps.TryInvokeUnaryOperator(context, sequence, "__length_hint__", out len_obj)) {
+                if (!(len_obj is NotImplementedType)) {
+                    hint = Converter.ConvertToInt32(len_obj);
+                    if (hint < 0) throw ValueError("__length_hint__() should return >= 0");
                     return true;
                 }
             }
@@ -4012,7 +4018,7 @@ internal static Exception TypeErrorForBadInstance(string template, object? insta
         }
 
         public static Exception TypeErrorForBinaryOp(string opSymbol, object? x, object? y) {
-            throw PythonOps.TypeError("unsupported operand type(s) for {0}: '{1}' and '{2}'",
+            throw PythonOps.TypeError("'{0}' not supported between instances of '{1}' and '{2}'",
                                 opSymbol, GetPythonTypeName(x), GetPythonTypeName(y));
         }
 

Src/IronPython/Runtime/PythonList.cs

@@ -740,7 +740,10 @@ public void extend([NotNone] PythonTuple/*!*/ seq) {
 
         public void extend(object? seq) {
             if (PythonOps.TryInvokeLengthHint(DefaultContext.Default, seq, out int len)) {
-                EnsureSize(len);
+                // CPython proceeds without resizing if the length overflows
+                if (int.MaxValue - len >= Count) {
+                    EnsureSize(Count + len);
+                }
             }
 
             ExtendNoLengthCheck(seq);