Buffer updates

CurrentVersion.props

@@ -3,9 +3,9 @@
   <PropertyGroup>
     <MajorVersion>2</MajorVersion>
     <MinorVersion>7</MinorVersion>
-    <MicroVersion>8</MicroVersion>
+    <MicroVersion>9</MicroVersion>
     <!-- Allowed: alpha,beta,candidate,final -->
-    <ReleaseLevel>final</ReleaseLevel>
+    <ReleaseLevel>candidate</ReleaseLevel>
     <ReleaseSerial>1</ReleaseSerial>
 
     <AssemblyRevision>0</AssemblyRevision>

Package/choco/IronPython.nuspec

@@ -23,5 +23,6 @@
     <file src="$STAGEDIR$\net45\DLLs\**" target="DLLs" exclude="**\*.xml" />
     <file src="$STAGEDIR$\Lib\**" target="Lib" />
     <file src="$STAGEDIR$\LICENSE;$STAGEDIR$\README.md" />
+    <file src="tools\*" target="tools" />
   </files>
 </package>

Package/choco/tools/VERIFICATION.txt

@@ -0,0 +1,3 @@
+VERIFICATION Verification is intended to assist the Chocolatey moderators and community in verifying that this package's contents are trustworthy.
+
+This package is published by the IronPython Project itself. The binaries are identical to other package types published by the project, in particular the IronPython nuget package.

Package/msi/IronPython.Installer.wixproj

@@ -79,6 +79,10 @@
       <HintPath>$(WixExtDir)\WixNetFxExtension.dll</HintPath>
       <Name>WixNetFxExtension</Name>
     </WixExtension>
+    <WixExtension Include="WixUtilExtension">
+      <HintPath>$(WixExtDir)\WixUtilExtension.dll</HintPath>
+      <Name>WixUtilExtension</Name>
+    </WixExtension>
   </ItemGroup>
   <ItemGroup>
     <HarvestDirectory Include="$(BindInputPaths)\Lib">

Src/IronPython.Modules/binascii.cs

@@ -359,14 +359,22 @@ public static Bytes hexlify(MemoryView data) {
         public static Bytes hexlify(Bytes data) {
             byte[] res = new byte[data.Count * 2];
             for (int i = 0; i < data.Count; i++) {
-
                 res[i * 2] = ToHex(data._bytes[i] >> 4);
                 res[(i * 2) + 1] = ToHex(data._bytes[i] & 0x0F);
             }
 
             return Bytes.Make(res);
         }
 
+        public static Bytes hexlify(ByteArray data) {
+            byte[] res = new byte[data.Count * 2];
+            for(int i =0; i < data.Count; i++) {
+                res[i * 2] = ToHex(data._bytes[i] >> 4);
+                res[(i * 2) + 1] = ToHex(data._bytes[i] & 0x0F);
+            }
+            return Bytes.Make(res);
+        }
+
         private static byte ToHex(int p) {
             if (p >= 10) {
                 return (byte)('a' + p - 10);
@@ -379,9 +387,9 @@ public static string hexlify([NotNull]PythonBuffer data) {
             return hexlify(data.ToString());
         }
 
-        public static object a2b_hex(CodeContext/*!*/ context, string data) {
+        public static object a2b_hex(CodeContext/*!*/ context, [BytesConversion] string data) {
             if (data == null) throw PythonOps.TypeError("expected string, got NoneType");
-            if ((data.Length & 0x01) != 0) throw Error(context, "string must be even lengthed");
+            if ((data.Length & 0x01) != 0) throw PythonOps.TypeError("Odd-length string");
             StringBuilder res = new StringBuilder(data.Length / 2);
 
             for (int i = 0; i < data.Length; i += 2) {
@@ -397,7 +405,7 @@ public static object a2b_hex(CodeContext/*!*/ context, string data) {
             return res.ToString();
         }
 
-        public static object unhexlify(CodeContext/*!*/ context, string hexstr) {
+        public static object unhexlify(CodeContext/*!*/ context, [BytesConversion] string hexstr) {
             return a2b_hex(context, hexstr);
         }
 

Src/IronPython/Runtime/ByteArray.cs

@@ -1,9 +1,11 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+
 using System.Collections;
 using System.Collections.Generic;
+using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -893,6 +895,23 @@ public PythonTuple __reduce__(CodeContext/*!*/ context) {
             return new ByteArray(bytes);
         }
 
+        /// <summary>
+        /// This overload is specifically implemented because in IronPython strings are unicode
+        /// </summary>
+        public static ByteArray operator +(ByteArray self, string other) {
+            List<byte> bytes;
+            lock(self) {
+                bytes = new List<byte>(self._bytes);
+            }
+
+            if(!other.TryMakeByteArray(out byte[] data)) {
+                throw PythonOps.TypeError("can't concat unicode to bytearray");
+            }
+
+            bytes.AddRange(data);
+            return new ByteArray(bytes);
+        }
+
         public static ByteArray operator *(ByteArray x, int y) {
             lock (x) {
                 if (y == 1) {

Src/IronPython/Runtime/MemoryView.cs

@@ -19,10 +19,16 @@ public MemoryView(IBufferProtocol @object) {
             _buffer = @object;
         }
 
-        public MemoryView(string @object) : this(Bytes.Make(@object.MakeByteArray())) { }
+        public MemoryView(IBufferProtocol @object, int start, int? end) : this(@object) {
+            _start = start;
+            _end = end;
+        }
 
-        private MemoryView(IBufferProtocol @object, int start, int? end) {
-            _buffer = @object;
+        public MemoryView(string @object) {
+            _buffer = new StringBufferProtocol(@object);
+        }
+
+        public MemoryView(string @object, int start, int? end) : this(@object) {
             _start = start;
             _end = end;
         }

Src/IronPython/Runtime/Operations/PythonOps.cs

@@ -3900,6 +3900,18 @@ public static byte[] MakeByteArray(this string s) {
             }
             return ret;
         }
+
+        public static bool TryMakeByteArray(this string s, out byte[] bytes) {
+            bytes = new byte[s.Length];
+            for (int i = 0; i < s.Length; i++) {
+                if(s[i] < 0x100) bytes[i] = (byte)s[i];
+                else {
+                    bytes = null;
+                    return false;
+                }
+            }
+            return true;
+        }
 
         public static string MakeString(this IList<byte> bytes) {
             return MakeString(bytes, bytes.Count);

Src/IronPython/Runtime/Operations/StringOps.cs

@@ -8,6 +8,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
+using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Text;
@@ -154,6 +155,70 @@ public virtual bool __contains__(object value) {
 
     }
 
+    [PythonHidden]
+    public class StringBufferProtocol : IBufferProtocol {
+        private string _wrapped;
+
+        public StringBufferProtocol(string wrapped) {
+            if(wrapped.Any(x => x > 255)) {
+                throw PythonOps.TypeError("cannot make memory view because object does not have the buffer interface");
+            }
+            _wrapped = wrapped;
+        }
+
+        int IBufferProtocol.ItemCount => _wrapped.Length;
+
+        string IBufferProtocol.Format => "B";
+
+        BigInteger IBufferProtocol.ItemSize => 1;
+
+        BigInteger IBufferProtocol.NumberDimensions => 1;
+
+        bool IBufferProtocol.ReadOnly => true;
+
+        PythonTuple IBufferProtocol.Strides => PythonTuple.MakeTuple(1);
+
+        object IBufferProtocol.SubOffsets => null;
+
+        Bytes IBufferProtocol.GetItem(int index) {
+            lock (this) {
+                return Bytes.Make(new byte[] { (byte)_wrapped[PythonOps.FixIndex(index, _wrapped.Length)] });
+            }
+        }
+
+        IList<BigInteger> IBufferProtocol.GetShape(int start, int? end) {
+            if (end != null) {
+                return new[] { (BigInteger)end - start };
+            }
+            return new[] { (BigInteger)_wrapped.Length - start };
+        }
+
+        void IBufferProtocol.SetItem(int index, object value) {
+            throw PythonOps.TypeError("cannot modify read-only memory");
+        }
+
+        void IBufferProtocol.SetSlice(Slice index, object value) {
+            throw PythonOps.TypeError("cannot modify read-only memory");
+        }
+
+        Bytes IBufferProtocol.ToBytes(int start, int? end) {
+            if (start == 0 && end == null) {
+                return new Bytes(_wrapped.MakeByteArray());
+            }
+
+            return new Bytes(StringOps.GetItem(_wrapped, new Slice(start, end)).MakeByteArray());
+        }
+
+        List IBufferProtocol.ToList(int start, int? end) {
+            string s = StringOps.GetItem(_wrapped, new Slice(start, end));
+            if (String.IsNullOrEmpty(s)) {
+                return new List();
+            }
+
+            return new List(s.MakeByteArray());
+        }
+    }
+
     /// <summary>
     /// StringOps is the static class that contains the methods defined on strings, i.e. 'abc'
     /// 
@@ -769,10 +834,7 @@ public static bool istitle(this string self) {
         }
 
         public static bool isunicode(this string self) {
-            foreach (char c in self) {
-                if (c >= LowestUnicodeValue) return true;
-            }
-            return false;
+            return self.Any(c => c >= LowestUnicodeValue);
         }
 
         /// <summary>

Src/IronPython/Runtime/PythonBuffer.cs

@@ -16,26 +16,17 @@
 
 using IronPython.Runtime.Binding;
 using IronPython.Runtime.Operations;
+using System.Numerics;
 
 namespace IronPython.Runtime {
     [PythonType("buffer"), DontMapGetMemberNamesToDir]
     public sealed class PythonBuffer : ICodeFormattable, IDynamicMetaObjectProvider, IList<byte> {
         internal object _object;
         private int _offset;
-        private int _size;
-
         private readonly CodeContext/*!*/ _context;
 
-        public PythonBuffer(CodeContext/*!*/ context, object @object)
-            : this(context, @object, 0) {
-        }
-
-        public PythonBuffer(CodeContext/*!*/ context, object @object, int offset)
-            : this(context, @object, offset, -1) {
-        }
-
         [Python3Warning("buffer() not supported in 3.x")]
-        public PythonBuffer(CodeContext/*!*/ context, object @object, int offset, int size) {
+        public PythonBuffer(CodeContext/*!*/ context, object @object, int offset=0, int size=-1) {
             PythonOps.Warn3k(context, "buffer() not supported in 3.x");
             if (!InitBufferObject(@object, offset, size)) {
                 throw PythonOps.TypeError("expected buffer object");
@@ -54,20 +45,17 @@ private bool InitBufferObject(object o, int offset, int size) {
             //  we currently support only buffers, strings and arrays
             //  of primitives, strings, bytes, and bytearray objects.
             int length;
-            if (o is PythonBuffer) {
-                PythonBuffer py = (PythonBuffer)o;
+            if (o is PythonBuffer py) {
                 o = py._object; // grab the internal object
-                length = py._size;
-            } else if (o is string) {
-                string strobj = (string)o;
+                length = py.Size;
+            } else if (o is string strobj) {
                 length = strobj.Length;
             } else if (o is Bytes) {
                 length = ((Bytes)o).Count;
             } else if (o is ByteArray) {
                 length = ((ByteArray)o).Count;
             } else if (o is Array || o is IPythonArray) {
-                Array arr = o as Array;
-                if (arr != null) {
+                if (o is Array arr) {
                     Type t = arr.GetType().GetElementType();
                     if (!t.IsPrimitive && t != typeof(string)) {
                         return false;
@@ -86,9 +74,9 @@ private bool InitBufferObject(object o, int offset, int size) {
 
             // reset the size based on the given buffer's original size
             if (size >= (length - offset) || size == -1) {
-                _size = length - offset;
+                Size = length - offset;
             } else {
-                _size = size;
+                Size = size;
             }
 
             _object = o;
@@ -129,13 +117,13 @@ public override bool Equals(object obj) {
         }
 
         public override int GetHashCode() {
-            return _object.GetHashCode() ^ _offset ^ (_size << 16 | (_size >> 16));
+            return _object.GetHashCode() ^ _offset ^ (Size << 16 | (Size >> 16));
         }
 
         private Slice GetSlice() {
             object end = null;
-            if (_size >= 0) {
-                end = _offset + _size;
+            if (Size >= 0) {
+                end = _offset + Size;
             }
             return new Slice(_offset, end);
         }
@@ -175,19 +163,16 @@ public object this[object s] {
         }
 
         private object GetSelectedRange() {
-            IPythonArray arr = _object as IPythonArray;
-            if (arr != null) {
+            if (_object is IPythonArray arr) {
                 return arr.tostring();
             }
 
-            ByteArray bytearr = _object as ByteArray;
-            if (bytearr != null) {
+            if (_object is ByteArray bytearr) {
                 return new Bytes((IList<byte>)bytearr[GetSlice()]);
             }
 
-            IPythonBufferable pyBuf = _object as IPythonBufferable;
-            if (pyBuf != null) {
-                return new Bytes(pyBuf.GetBytes(_offset, _size));
+            if (_object is IPythonBufferable pyBuf) {
+                return new Bytes(pyBuf.GetBytes(_offset, Size));
             }
 
             return PythonOps.GetIndex(_context, _object, GetSlice());
@@ -228,20 +213,16 @@ private object GetSelectedRange() {
         }
 
         public int __len__() {
-            return Math.Max(_size, 0);
+            return Math.Max(Size, 0);
         }
 
-        internal int Size {
-            get {
-                return _size;
-            }
-        }
+        internal int Size { get; private set; }
 
         #region ICodeFormattable Members
 
         public string/*!*/ __repr__(CodeContext/*!*/ context) {
             return string.Format("<read-only buffer for 0x{0:X16}, size {1}, offset {2} at 0x{3:X16}>",
-                PythonOps.Id(_object), _size, _offset, PythonOps.Id(this));
+                PythonOps.Id(_object), Size, _offset, PythonOps.Id(this));
         }
 
         #endregion
@@ -374,7 +355,8 @@ bool ICollection<byte>.IsReadOnly {
                 return true;
             }
         }
-        #endregion
+
+        #endregion       
     }
 
     /// <summary>

make.ps1

@@ -25,6 +25,11 @@ if(!$global:isUnix) {
         Exit 1
     }
 
+    if(-not [System.IO.Directory]::Exists($_VSINSTPATH)) {
+        Write-Error "Could not determine installation path to Visual Studio"
+        Exit 1
+    }
+
     if([System.IO.File]::Exists([System.IO.Path]::Combine($_VSINSTPATH, 'MSBuild\15.0\Bin\MSBuild.exe'))) {
         $env:PATH = [String]::Join(';', $env:PATH, [System.IO.Path]::Combine($_VSINSTPATH, 'MSBuild\15.0\Bin'))
     }