Factor out IPythonBuffer from IBufferProtocol

IronPythonAnalyzer/IronPythonAnalyzer/IronPythonAnalyzerAnalyzer.cs

@@ -51,16 +51,15 @@ public class IronPythonAnalyzerAnalyzer : DiagnosticAnalyzer {
                 var bytesLikeAttributeSymbol = context.Compilation.GetTypeByMetadataName("IronPython.Runtime.BytesLikeAttribute");
 
                 var byteType = context.Compilation.GetTypeByMetadataName("System.Byte");
-                var readOnlyMemoryType = context.Compilation.GetTypeByMetadataName("System.ReadOnlyMemory`1");
                 var ireadOnlyListType = context.Compilation.GetTypeByMetadataName("System.Collections.Generic.IReadOnlyList`1");
                 var ilistType = context.Compilation.GetTypeByMetadataName("System.Collections.Generic.IList`1");
-                var readOnlyMemoryOfByteType = readOnlyMemoryType.Construct(byteType);
                 var ireadOnlyListOfByteType = ireadOnlyListType.Construct(byteType);
                 var ilistOfByteType = ilistType.Construct(byteType);
+                var ibufferProtocolType = context.Compilation.GetTypeByMetadataName("IronPython.Runtime.IBufferProtocol");
 
                 foreach (IParameterSymbol parameterSymbol in methodSymbol.Parameters) {
                     if (parameterSymbol.GetAttributes().Any(x => x.AttributeClass.Equals(bytesLikeAttributeSymbol))
-                        && !parameterSymbol.Type.Equals(readOnlyMemoryOfByteType)
+                        && !parameterSymbol.Type.Equals(ibufferProtocolType)
                         && !parameterSymbol.Type.Equals(ireadOnlyListOfByteType) && !parameterSymbol.Type.Equals(ilistOfByteType)) {
                         var diagnostic = Diagnostic.Create(Rule3, parameterSymbol.Locations[0], parameterSymbol.Name, parameterSymbol.Type.MetadataName);
                         context.ReportDiagnostic(diagnostic);

Src/IronPython.Modules/IronPython.Modules.csproj

@@ -5,6 +5,7 @@
     <BaseAddress>885063680</BaseAddress>
     <CodeAnalysisRuleSet>..\..\IronPython.ruleset</CodeAnalysisRuleSet>
     <DocumentationFile>$(OutputPath)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup Condition=" '$(IsFullFramework)' == 'true' ">

Src/IronPython.Modules/_ctypes/Array.cs

@@ -21,7 +21,7 @@ namespace IronPython.Modules {
     /// Provides support for interop with native code from Python code.
     /// </summary>
     public static partial class CTypes {
-        
+
         [PythonType("Array")]
         public abstract class _Array : CData {
 
@@ -127,7 +127,7 @@ public abstract class _Array : CData {
                 }
             }
 
-            
+
             internal override PythonTuple GetBufferInfo() {
                 INativeType elemType = ElementType;
                 int dimensions = 1;
@@ -179,6 +179,8 @@ public abstract class _Array : CData {
                 return shape;
             }
 
+            // TODO: if this.NativeType.ElementType is ArrayType, suboffsets need to be reported
+
 #endregion
         }
     }

Src/IronPython.Modules/_ctypes/CData.cs

@@ -12,6 +12,7 @@
 using IronPython.Runtime;
 using IronPython.Runtime.Types;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace IronPython.Modules {
     /// <summary>
@@ -35,7 +36,7 @@ public static partial class CTypes {
         /// Base class for all ctypes interop types.
         /// </summary>
         [PythonType("_CData"), PythonHidden]
-        public abstract class CData : IBufferProtocol {
+        public abstract class CData : IBufferProtocol, IPythonBuffer {
             internal MemoryHolder _memHolder;
 
             // members: __setstate__,  __reduce__ _b_needsfree_ __ctypes_from_outparam__ __hash__ _objects _b_base_ __doc__
@@ -95,17 +96,21 @@ public abstract class CData : IBufferProtocol {
 
             #region IBufferProtocol Members
 
-            object IBufferProtocol.GetItem(int index) {
-                return Bytes.Make(GetBytes(index, NativeType.Size));
+            IPythonBuffer IBufferProtocol.GetBuffer(BufferFlags flags) {
+                return this;
             }
 
-            void IBufferProtocol.SetItem(int index, object value) {
-                throw new NotImplementedException();
-            }
+            void IDisposable.Dispose() { }
 
-            void IBufferProtocol.SetSlice(Slice index, object value) {
-                throw new NotImplementedException();
-            }
+            object IPythonBuffer.Object => this;
+
+            unsafe ReadOnlySpan<byte> IPythonBuffer.AsReadOnlySpan()
+                => new Span<byte>(_memHolder.UnsafeAddress.ToPointer(), _memHolder.Size);
+
+            unsafe Span<byte> IPythonBuffer.AsSpan()
+                => new Span<byte>(_memHolder.UnsafeAddress.ToPointer(), _memHolder.Size);
+
+            int IPythonBuffer.Offset => 0;
 
             public virtual int ItemCount {
                 [PythonHidden]
@@ -114,46 +119,43 @@ public abstract class CData : IBufferProtocol {
                 }
             }
 
-            string IBufferProtocol.Format {
+            string IPythonBuffer.Format {
                 get { return NativeType.TypeFormat; }
             }
 
+            // TODO: change sig
             public virtual BigInteger ItemSize {
                 [PythonHidden]
                 get { return this.NativeType.Size; }
             }
 
-            BigInteger IBufferProtocol.NumberDimensions {
-                get { return 0; }
+            int IPythonBuffer.ItemSize => (int)this.ItemSize;
+
+            int IPythonBuffer.NumOfDims {
+                get {
+                    return GetShape(0, null)?.Count ?? (ItemCount > 1 ? 1 : 0);
+                }
             }
 
-            bool IBufferProtocol.ReadOnly {
+            bool IPythonBuffer.IsReadOnly {
                 get { return false; }
             }
 
+            // TODO: change sig
             [PythonHidden]
             public virtual IList<BigInteger> GetShape(int start, int? end) {
                 return null;
             }
 
-            PythonTuple IBufferProtocol.Strides {
-                get { return null; }
-            }
-
-            PythonTuple IBufferProtocol.SubOffsets {
-                get { return null; }
-            }
+            IReadOnlyList<int> IPythonBuffer.Shape => GetShape(0, null)?.Select(big => (int)big).ToArray();  // TODO: remove using Linq when done with sig change
 
-            Bytes IBufferProtocol.ToBytes(int start, int? end) {
-                return Bytes.Make(GetBytes(start, NativeType.Size));
-            }
 
-            PythonList IBufferProtocol.ToList(int start, int? end) {
-                return new PythonList(((IBufferProtocol)this).ToBytes(start, end));
+            IReadOnlyList<int> IPythonBuffer.Strides {
+                get { return null; }
             }
 
-            ReadOnlyMemory<byte> IBufferProtocol.ToMemory() {
-                return GetBytes(0, NativeType.Size);
+            IReadOnlyList<int> IPythonBuffer.SubOffsets {
+                get { return null; }
             }
 
             #endregion

Src/IronPython.Modules/_operator.cs

@@ -352,14 +352,14 @@ public class methodcaller {
                 string aStr = a as string;
                 string bStr = b as string;
                 return CompareBytes(aStr.MakeByteArray(), bStr.MakeByteArray());
-            } else if(a is IBufferProtocol && b is IBufferProtocol) {
-                IBufferProtocol aBuf = a as IBufferProtocol;
-                IBufferProtocol bBuf = b as IBufferProtocol;
-                if(aBuf.NumberDimensions > 1 || bBuf.NumberDimensions > 1) {
+            } else if(a is IBufferProtocol abp && b is IBufferProtocol bbp) {
+                using IPythonBuffer aBuf = abp.GetBuffer();
+                using IPythonBuffer bBuf = bbp.GetBuffer();
+                if(aBuf.NumOfDims > 1 || bBuf.NumOfDims > 1) {
                     throw PythonOps.BufferError("Buffer must be single dimension");
                 }
 
-                return CompareBytes(aBuf.ToBytes(0, null), bBuf.ToBytes(0, null));
+                return CompareBytes(aBuf.AsReadOnlySpan().ToArray(), bBuf.AsReadOnlySpan().ToArray());
             }
             throw PythonOps.TypeError("unsupported operand types(s) or combination of types: '{0}' and '{1}", PythonOps.GetPythonTypeName(a), PythonOps.GetPythonTypeName(b));
         }

Src/IronPython.Modules/_ssl.cs

@@ -181,8 +181,11 @@ public class _SSLContext {
                     var cabuf = cadata as IBufferProtocol;
                     if (cabuf != null) {
                         int pos = 0;
-                        byte[] contents = cabuf.ToBytes(0, null).ToArray();
-                        while(pos < contents.Length) {
+                        byte[] contents;
+                        using (IPythonBuffer buf = cabuf.GetBuffer()) {
+                            contents = buf.AsReadOnlySpan().ToArray();
+                        }
+                        while (pos < contents.Length) {
                             byte[] curr = new byte[contents.Length - pos];
                             Array.Copy(contents, pos, curr, 0, contents.Length - pos);
                             var cert = new X509Certificate2(curr);

Src/IronPython.Modules/binascii.cs

@@ -680,8 +680,11 @@ public static Bytes unhexlify(CodeContext/*!*/ context, [NotNull]string hexstr)
             return ret;
         }
 
-        private static IList<byte> tobytes(this IBufferProtocol buffer)
-            => buffer as IList<byte> ?? buffer.ToBytes(0, null);
+        private static IList<byte> tobytes(this IBufferProtocol buffer) {
+            if (buffer is IList<byte> list) return list;
+
+            return new Bytes(buffer);
+        }
 
         #endregion
     }

Src/IronPython.Modules/re.cs

@@ -450,7 +450,9 @@ public override bool Equals(object obj)
                 if (pattern is Bytes) {
                     switch (@string) {
                         case IBufferProtocol bufferProtocol:
-                            str = bufferProtocol.ToBytes(0, null).MakeString();
+                            using (IPythonBuffer buf = bufferProtocol.GetBuffer()) {
+                                str = buf.AsReadOnlySpan().MakeString();
+                            }
                             break;
                         case IList<byte> b:
                             str = b.MakeString();
@@ -494,7 +496,9 @@ public override bool Equals(object obj)
                 if (pattern is Bytes) {
                     switch (repl) {
                         case IBufferProtocol bufferProtocol:
-                            str = bufferProtocol.ToBytes(0, null).MakeString();
+                            using (IPythonBuffer buf = bufferProtocol.GetBuffer()) {
+                                str = buf.AsReadOnlySpan().MakeString();
+                            }
                             break;
                         case IList<byte> b:
                             str = b.MakeString();
@@ -1231,7 +1235,9 @@ internal class ParsedRegex {
             if (pattern is Bytes) {
                 switch (@string) {
                     case IBufferProtocol bufferProtocol:
-                        str = bufferProtocol.ToBytes(0, null).MakeString();
+                        using (IPythonBuffer buf = bufferProtocol.GetBuffer()) {
+                            str = buf.AsReadOnlySpan().MakeString();
+                        }
                         break;
                     case IList<byte> b:
                         str = b.MakeString();

Src/IronPython/Modules/_bytesio.cs

@@ -43,7 +43,7 @@ public BytesIO(CodeContext/*!*/ context, object initial_bytes=null)
                 : base(context) {
             }
 
-            public void __init__(object initial_bytes=null) {
+            public void __init__(IBufferProtocol initial_bytes=null) {
                 if (Object.ReferenceEquals(_data, null)) {
                     _data = new byte[DEFAULT_BUF_SIZE];
                 }
@@ -94,8 +94,8 @@ public BytesIO(CodeContext/*!*/ context, object initial_bytes=null)
 
             public MemoryView getbuffer() {
                 _checkClosed();
-
-                return new MemoryView(new Bytes(_data), 0, _length, 1, "B", PythonTuple.MakeTuple(_length), readonlyView: true);
+                // TODO: MemoryView constructor should accept Memory/ReadOnlyMemory
+                return (MemoryView)new MemoryView(new Bytes(_data))[new Slice(0, _length, 1)];
             }
 
             [Documentation("isatty() -> False\n\n"
@@ -355,7 +355,12 @@ public BytesIO(CodeContext/*!*/ context, object initial_bytes=null)
                 )]
             public override BigInteger write(CodeContext/*!*/ context, object bytes) {
                 _checkClosed();
+                if (bytes is IBufferProtocol bufferProtocol) return DoWrite(bufferProtocol);
+                throw PythonOps.TypeError("a bytes-like object is required, not '{0}'", PythonTypeOps.GetName(bytes));
+            }
 
+            public BigInteger write(CodeContext/*!*/ context, IBufferProtocol bytes) {
+                _checkClosed();
                 return DoWrite(bytes);
             }
 
@@ -418,83 +423,24 @@ public BytesIO(CodeContext/*!*/ context, object initial_bytes=null)
 
             #region Private implementation details
 
-            private int DoWrite(byte[] bytes) {
+            private int DoWrite(IBufferProtocol bufferProtocol) {
+                using var buffer = bufferProtocol.GetBuffer();
+                var bytes = buffer.AsReadOnlySpan();
+
                 if (bytes.Length == 0) {
                     return 0;
                 }
 
                 EnsureSizeSetLength(_pos + bytes.Length);
-                Array.Copy(bytes, 0, _data, _pos, bytes.Length);
+                bytes.CopyTo(_data.AsSpan(_pos, bytes.Length));
 
                 _pos += bytes.Length;
                 return bytes.Length;
             }
 
-            private int DoWrite(ICollection<byte> bytes) {
-                int nbytes = bytes.Count;
-                if (nbytes == 0) {
-                    return 0;
-                }
-
-                EnsureSizeSetLength(_pos + nbytes);
-                bytes.CopyTo(_data, _pos);
-
-                _pos += nbytes;
-                return nbytes;
-            }
-
-            private int DoWrite(string bytes) {
-                // CLR strings are natively Unicode (UTF-16 LE, to be precise).
-                // In 2.x, io.BytesIO.write() takes "bytes or bytearray" as a parameter.
-                // On 2.x "bytes" is an alias for str, so str types are accepted by BytesIO.write().
-                // When given a unicode object, 2.x BytesIO.write() complains:
-                //   TypeError: 'unicode' does not have the buffer interface
-
-                // We will accept CLR strings, but only if the data in it is in Latin 1 (iso-8859-1)
-                // encoding (i.e. ord(c) for all c is within 0-255.
-
-                // Alternatively, we could support strings containing any Unicode character by ignoring
-                // any 0x00 bytes, but as CPython doesn't support that it is unlikely that we will need to.
-
-                int nbytes = bytes.Length;
-                if (nbytes == 0) {
-                    return 0;
-                }
-
-                byte[] _raw_string = new byte[nbytes];
-                for (int i = 0; i < nbytes; i++) {
-                    int ord = (int)bytes[i];
-                    if(ord < 256) {
-                        _raw_string[i] = (byte)ord;
-                    } else {
-                        // A character outside the range 0x00-0xFF is present in the original string.
-                        // Ejecting, emulating the cPython 2.x behavior when it enounters "unicode".
-                        // This should keep the unittest gods at bay.
-                        throw PythonOps.TypeError("'unicode' does not have the buffer interface");
-                    }
-                }
-
-                return DoWrite(_raw_string);
-            }
-
             private int DoWrite(object bytes) {
-                switch (bytes) {
-                    case byte[] b:
-                        return DoWrite(b);
-                    case Bytes b:
-                        return DoWrite(b.UnsafeByteArray);
-                    case ArrayModule.array a:
-                        return DoWrite(a.ToByteArray()); // as byte[]
-                    case ICollection<byte> c:
-                        return DoWrite(c);
-                    case string s:
-                        // TODO Remove this when we move to 3.x
-                        return DoWrite(s);
-                    case MemoryView mv:
-                        return DoWrite(mv.tobytes().UnsafeByteArray);
-                }
-
-                throw PythonOps.TypeError("expected a readable buffer object");
+                if (bytes is IBufferProtocol bufferProtocol) return DoWrite(bufferProtocol);
+                return DoWrite(Converter.Convert<IBufferProtocol>(bytes));
             }
 
             private void EnsureSize(int size) {

Src/IronPython/Modules/_io.cs

@@ -421,12 +421,11 @@ public class _RawIOBase : _IOBase, IDynamicMetaObjectProvider {
                 ByteArray arr = new ByteArray(new byte[sizeInt]);
                 sizeInt = (int)readinto(context, arr);
 
-                var res = arr.UnsafeByteList;
-                if (sizeInt < res.Count) {
-                    res.RemoveRange(sizeInt, res.Count - sizeInt);
-                }
-
-                return new Bytes(res);
+                // cannot use arr.UnsafeByteList.RemoveRange(sizeInt, res.Count - sizeInt)
+                // because HTTPResponse.client.readinto may wrap `arr` in `memoryview`
+                // and never release it
+                using var buf = ((IBufferProtocol)arr).GetBuffer();
+                return Bytes.Make(buf.AsReadOnlySpan().Slice(0, sizeInt).ToArray());
             }
 
             public Bytes readall(CodeContext/*!*/ context) {