Backport more changes

Src/IronPython.Modules/_collections.cs

@@ -551,20 +551,13 @@ public void __delitem__(CodeContext/*!*/ context, object index) {
                 }
             }
 
-            public PythonTuple __reduce__() {
-                lock (_lockObj) {
-                    object[] items = new object[_itemCnt];
-                    int curItem = 0;
-                    WalkDeque(delegate(int curIndex) {
-                        items[curItem++] = _data[curIndex];
-                        return true;
-                    });
-
-                    return PythonTuple.MakeTuple(
-                        DynamicHelpers.GetPythonType(this),
-                        PythonTuple.MakeTuple(PythonList.FromArrayNoCopy(items))
-                    );
-                }
+            public PythonTuple __reduce__(CodeContext context) {
+                return PythonTuple.MakeTuple(
+                    DynamicHelpers.GetPythonType(this),
+                    _maxLen == -1 ? PythonTuple.EMPTY : PythonTuple.MakeTuple(PythonTuple.EMPTY, maxlen),
+                    GetType() == typeof(deque) ? null : PythonOps.GetBoundAttr(context, this, "__dict__"),
+                    PythonOps.GetEnumeratorObject(context, this)
+                );
             }
 
             public int __len__() {

Src/IronPython.Modules/_socket.cs

@@ -136,14 +136,27 @@ public void __init__(CodeContext/*!*/ context, int family = DefaultAddressFamily
                     throw MakeException(context, new SocketException((int)SocketError.ProtocolNotSupported));
                 }
 
-                Socket? socket;
+                Socket? socket = null;
                 if (fileno is socket sock) {
                     socket = sock._socket;
                     _hostName = sock._hostName;
                     // we now own the lifetime of the socket
                     GC.SuppressFinalize(sock);
-                } else if (fileno != null && (socket = HandleToSocket((long)fileno)) != null) {
-                    // nothing to do here
+                } else if (fileno != null) {
+                    if (!PythonOps.TryToIndex(fileno, out object? handleObj)) {
+                        throw PythonOps.TypeErrorForUnIndexableObject(fileno);
+                    }
+                    long handle = Converter.ConvertToInt64(handleObj);
+                    // Windows reserves only INVALID_SOCKET (~0) as an invalid handle
+                    if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? handle == -1 : handle < 0) {
+                        throw PythonOps.ValueError("negative file descriptor");
+                    }
+                    socket = HandleToSocket(handle);
+                    if (socket is null) {
+                        throw PythonOps.OSError(RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+                            ? PythonErrorNumber.WSAENOTSOCK : PythonErrorNumber.EBADF,
+                            "Bad file descriptor");
+                    }
                 } else {
                     try {
                         socket = new Socket(addressFamily, socketType, protocolType);
@@ -413,7 +426,7 @@ public Bytes recv(int bufsize, int flags = 0) {
                     try {
                         bytesRead = _socket.Receive(buffer, bufsize, (SocketFlags)flags);
                     } catch (Exception e) {
-                        throw MakeRecvException(e, SocketError.NotConnected);
+                        throw MakeException(_context, e);
                     }
 
                     var bytes = new byte[bytesRead];
@@ -449,7 +462,7 @@ public int recv_into([NotNone] IBufferProtocol buffer, int nbytes = 0, int flags
                     try {
                         bytesRead = _socket.Receive(byteBuffer, nbytes, (SocketFlags)flags);
                     } catch (Exception e) {
-                        throw MakeRecvException(e, SocketError.NotConnected);
+                        throw MakeException(_context, e);
                     }
 
                     byteBuffer.AsSpan(0, bytesRead).CopyTo(span);
@@ -486,7 +499,7 @@ public PythonTuple recvfrom(int bufsize, int flags = 0) {
                     try {
                         bytesRead = _socket.ReceiveFrom(buffer, bufsize, (SocketFlags)flags, ref remoteEP);
                     } catch (Exception e) {
-                        throw MakeRecvException(e, SocketError.InvalidArgument);
+                        throw MakeException(_context, e);
                     }
 
                     var bytes = new byte[bytesRead];
@@ -519,7 +532,7 @@ public PythonTuple recvfrom_into([NotNone] IBufferProtocol buffer, int nbytes =
                     try {
                         bytesRead = _socket.ReceiveFrom(byteBuffer, nbytes, (SocketFlags)flags, ref remoteEP);
                     } catch (Exception e) {
-                        throw MakeRecvException(e, SocketError.InvalidArgument);
+                        throw MakeException(_context, e);
                     }
 
                     byteBuffer.AsSpan(0, bytesRead).CopyTo(span);
@@ -551,18 +564,6 @@ private static int byteBufferSize(string funcName, int nbytes, int bufLength, in
                 }
             }
 
-            private Exception MakeRecvException(Exception e, SocketError errorCode = SocketError.InvalidArgument) {
-                if (e is ObjectDisposedException) return MakeException(_context, e);
-
-                // on the socket recv throw a special socket error code when SendTimeout is zero
-                if (_socket.SendTimeout == 0) {
-                    var s = new SocketException((int)errorCode);
-                    return PythonExceptions.CreateThrowable(error, s.ErrorCode, s.Message);
-                } else {
-                    return MakeException(_context, e);
-                }
-            }
-
             [Documentation("send(string[, flags]) -> bytes_sent\n\n"
                 + "Send data to the remote socket. The socket must be connected to a remote\n"
                 + "socket (by calling either connect() or accept(). Returns the number of bytes\n"
@@ -1787,7 +1788,7 @@ internal static Exception MakeException(CodeContext/*!*/ context, Exception exce
                 }
             } else if (exception is ObjectDisposedException) {
                 return PythonExceptions.CreateThrowable(error, PythonErrorNumber.EBADF, "Socket is closed");
-            } else if (exception is InvalidOperationException) {
+            } else if (exception is InvalidOperationException or ArgumentException) {
                 return MakeException(context, new SocketException((int)SocketError.InvalidArgument));
             } else {
                 return exception;

Src/IronPython.Modules/array.cs

@@ -196,11 +196,7 @@ public array InPlaceMultiply(int value) {
                 if (value <= 0) {
                     _data.Clear();
                 } else {
-                    var myData = __copy__();
-
-                    for (int i = 0; i < (value - 1); i++) {
-                        ExtendArray(myData);
-                    }
+                    _data.InPlaceMultiply(value);
                 }
                 return this;
             }

Src/IronPython.Modules/time.cs

@@ -340,10 +340,12 @@ private static List<FormatInfo> PythonFormatToCLIFormat(string format) {
             List<FormatInfo> newFormat = new List<FormatInfo>();
 
             for (int i = 0; i < format.Length; i++) {
-                if (format[i] == '%') {
+                var ch = format[i];
+                if (ch == '%') {
                     if (i + 1 == format.Length) throw PythonOps.ValueError("Invalid format string");
 
                     switch (format[++i]) {
+                        case '\0': throw PythonOps.ValueError("embedded null character");
                         case 'a': newFormat.Add(new FormatInfo("ddd")); break;
                         case 'A': newFormat.Add(new FormatInfo("dddd")); break;
                         case 'b': newFormat.Add(new FormatInfo("MMM")); break;
@@ -379,10 +381,11 @@ private static List<FormatInfo> PythonFormatToCLIFormat(string format) {
                             throw PythonOps.ValueError("Invalid format string");
                     }
                 } else {
+                    if (ch == '\0') throw PythonOps.ValueError("embedded null character");
                     if (newFormat.Count == 0 || newFormat[newFormat.Count - 1].Type != FormatInfoType.UserText)
-                        newFormat.Add(new FormatInfo(FormatInfoType.UserText, format[i].ToString()));
+                        newFormat.Add(new FormatInfo(FormatInfoType.UserText, ch.ToString()));
                     else
-                        newFormat[newFormat.Count - 1].Text += format[i];
+                        newFormat[newFormat.Count - 1].Text += ch;
                 }
             }
 

Src/IronPython.Modules/winreg.cs

@@ -4,9 +4,8 @@
 
 #if FEATURE_REGISTRY
 
-using Microsoft.Win32;
-using Microsoft.Win32.SafeHandles;
 using System;
+using System.Buffers.Binary;
 using System.Collections.Concurrent;
 using System.ComponentModel;
 using System.Diagnostics;
@@ -22,6 +21,9 @@
 using IronPython.Runtime.Exceptions;
 using IronPython.Runtime.Types;
 
+using Microsoft.Win32;
+using Microsoft.Win32.SafeHandles;
+
 [assembly: PythonModule("winreg", typeof(IronPython.Modules.PythonWinReg), PlatformsAttribute.PlatformFamily.Windows)]
 namespace IronPython.Modules {
     [SupportedOSPlatform("windows")]
@@ -70,6 +72,7 @@ public static class PythonWinReg {
         public const int REG_FULL_RESOURCE_DESCRIPTOR = 0X9;
         public const int REG_RESOURCE_REQUIREMENTS_LIST = 0XA;
         public const int REG_QWORD = 0xB;
+        public const int REG_QWORD_LITTLE_ENDIAN = 0XB;
 
         public const int REG_NOTIFY_CHANGE_NAME = 0X1;
         public const int REG_NOTIFY_CHANGE_ATTRIBUTES = 0X2;
@@ -346,19 +349,22 @@ private static void QueryValueExImpl(SafeRegistryHandle handle, string valueName
                     break;
                 case REG_EXPAND_SZ:
                 case REG_SZ:
-                    if (length >= 2 && data[length - 1] == 0 && data[length - 2] == 0) {
-                        value = ExtractString(data, 0, (int)length - 2);
-                    } else {
-                        value = ExtractString(data, 0, (int)length);
-                    }
+                    var span = MemoryMarshal.Cast<byte, char>(data.AsSpan());
+                    var len = span.IndexOf((char)0);
+                    if (len != -1) span = span.Slice(0, len);
+                    value = span.ToString();
                     break;
                 case REG_DWORD:
-                    if (BitConverter.IsLittleEndian) {
-                        value = (uint)((data[3] << 24) | (data[2] << 16) | (data[1] << 8) | data[0]);
-                    } else {
-                        value = (uint)((data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]);
-                    }
+                    var dval = BitConverter.ToUInt32(data, 0);
+                    if (!BitConverter.IsLittleEndian) dval = BinaryPrimitives.ReverseEndianness(dval);
+                    value = dval > int.MaxValue ? (BigInteger)dval : unchecked((int)dval);
                     break;
+                case REG_QWORD:
+                    var qval = BitConverter.ToUInt64(data, 0);
+                    if (!BitConverter.IsLittleEndian) qval = BinaryPrimitives.ReverseEndianness(qval);
+                    value = (BigInteger)qval;
+                    break;
+
                 default:
                     value = null;
                     break;

Src/IronPython.Modules/winsound.cs

@@ -120,20 +120,20 @@ public static void PlaySound(CodeContext/*!*/ context, [NotNone] IBufferProtocol
 This parameter must be in the range 37 through 32,767.
 The duration argument specifies the number of milliseconds.
 ")]
-        public static void Beep(CodeContext/*!*/ context, int freq, int dur) {
-            if (freq < 37 || freq > 32767) {
+        public static void Beep(CodeContext/*!*/ context, int frequency, int duration) {
+            if (frequency < 37 || frequency > 32767) {
                 throw PythonOps.ValueError("frequency must be in 37 thru 32767");
             }
 
-            bool ok = Beep(freq, dur);
+            bool ok = Beep(frequency, duration);
             if (!ok) {
                 throw PythonOps.RuntimeError("Failed to beep");
             }
         }
 
         [Documentation("MessageBeep(x) - call Windows MessageBeep(x). x defaults to MB_OK.")]
-        public static void MessageBeep(CodeContext/*!*/ context, int x=MB_OK) {
-            MessageBeep(x);
+        public static void MessageBeep(CodeContext/*!*/ context, int type=MB_OK) {
+            MessageBeep(type);
         }
 
         #endregion

Src/IronPython/Lib/iptest/ipunittest.py

@@ -206,24 +206,25 @@ def _flatten_suite(suite):
 
 def generate_suite(tests, failing_tests, skip_tests=[]):
     all_tests = _flatten_suite(tests)
+    test_indices = {t: i for i, t in enumerate(all_tests)}
     unknown_tests = []
 
     for t in skip_tests:
         try:
-            all_tests.remove(t)
+            all_tests[test_indices.pop(t)] = None
         except ValueError:
             unknown_tests.append(t)
 
     for t in failing_tests:
         try:
-            all_tests[all_tests.index(t)] = unittest.expectedFailure(t)
-        except:
+            all_tests[test_indices.pop(t)] = unittest.expectedFailure(t)
+        except ValueError:
             unknown_tests.append(t)
 
     if unknown_tests:
         raise ValueError("Unknown tests:\n - {}".format('\n - '.join(str(t) for t in unknown_tests)))
 
-    return unittest.TestSuite(all_tests)
+    return unittest.TestSuite(t for t in all_tests if t is not None)
 
 # class testpath:
 #     # find the ironpython root directory

Src/IronPython/Modules/Builtin.cs

@@ -135,7 +135,7 @@ public static string bin(object? number) {
 
         public static PythonType bytes => TypeCache.Bytes;
 
-        public static PythonType bytearray => DynamicHelpers.GetPythonTypeFromType(typeof(ByteArray));
+        public static PythonType bytearray => TypeCache.ByteArray;
 
         [Documentation("callable(object) -> bool\n\nReturn whether the object is callable (i.e., some kind of function).")]
         public static bool callable(CodeContext/*!*/ context, object? o) {

Src/IronPython/Runtime/ArrayData.cs

@@ -27,6 +27,7 @@ internal interface ArrayData : IList {
         void InsertRange(int index, int count, ArrayData value);
         void RemoveSlice(Slice slice);
         ArrayData Multiply(int count);
+        void InPlaceMultiply(int count);
         new bool Remove(object? item);
         void Reverse();
         Span<byte> AsByteSpan();

Src/IronPython/Runtime/ByteArray.cs

@@ -9,16 +9,16 @@
 using System.Collections.Generic;
 using System.Numerics;
 using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
 using System.Text;
 
-using Microsoft.Scripting.Runtime;
-using Microsoft.Scripting.Utils;
-
 using IronPython.Runtime.Exceptions;
 using IronPython.Runtime.Operations;
 using IronPython.Runtime.Types;
 
+using Microsoft.Scripting;
+using Microsoft.Scripting.Runtime;
+using Microsoft.Scripting.Utils;
+
 namespace IronPython.Runtime {
     /// <summary>
     /// bytearray(string, encoding[, errors]) -> bytearray
@@ -56,6 +56,12 @@ internal ByteArray(IEnumerable<byte> bytes) {
             _bytes = new ArrayData<byte>(bytes);
         }
 
+        [StaticExtensionMethod]
+        public static object __new__(CodeContext context, [NotNone] PythonType cls, [ParamDictionary, NotNone] IDictionary<object, object> dict, [NotNone] params object[] args) {
+            if (cls == TypeCache.ByteArray) return new ByteArray();
+            return cls.CreateInstance(context);
+        }
+
         public void __init__() {
             lock (this) {
                 _bytes.Clear();
@@ -74,7 +80,7 @@ public void __init__(int source) {
         }
 
         public void __init__([NotNone] IBufferProtocol source) {
-            if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) {
+            if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) {
                 __init__(size);
             } else {
                 lock (this) {
@@ -86,7 +92,7 @@ public void __init__([NotNone] IBufferProtocol source) {
         }
 
         public void __init__(CodeContext context, object? source) {
-            if (Converter.TryConvertToIndex(source, out int size, throwNonInt: false)) {
+            if (Converter.TryConvertToIndex(source, out int size, throwTypeError: false)) {
                 __init__(size);
             } else if (source is IEnumerable<byte> en) {
                 lock (this) {
@@ -466,8 +472,13 @@ public int find(BigInteger @byte, object? start, object? end) {
             }
         }
 
-        public static ByteArray fromhex([NotNone] string @string) {
-            return new ByteArray(IListOfByteOps.FromHex(@string));
+        [ClassMethod]
+        public static object fromhex(CodeContext context, [NotNone] PythonType cls, [NotNone] string @string) {
+            var hex = IListOfByteOps.FromHex(@string);
+            if (cls == TypeCache.ByteArray) {
+                return new ByteArray(hex);
+            }
+            return PythonTypeOps.CallParams(context, cls, new Bytes(hex));
         }
 
         public string hex() => Bytes.ToHex(_bytes.AsByteSpan()); // new in CPython 3.5