Implement Interlocked for small types.

src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs

@@ -42,6 +42,24 @@ public static partial class Interlocked
         #endregion
 
         #region Exchange
+        /// <summary>Sets a 8-bit unsigned integer to a specified value and returns the original value, as an atomic operation.</summary>
+        /// <param name="location1">The variable to set to the specified value.</param>
+        /// <param name="value">The value to which the <paramref name="location1"/> parameter is set.</param>
+        /// <returns>The original value of <paramref name="location1"/>.</returns>
+        /// <exception cref="NullReferenceException">The address of location1 is a null pointer.</exception>
+        [Intrinsic]
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        public static extern byte Exchange(ref byte location1, byte value);
+
+        /// <summary>Sets a 16-bit signed integer to a specified value and returns the original value, as an atomic operation.</summary>
+        /// <param name="location1">The variable to set to the specified value.</param>
+        /// <param name="value">The value to which the <paramref name="location1"/> parameter is set.</param>
+        /// <returns>The original value of <paramref name="location1"/>.</returns>
+        /// <exception cref="NullReferenceException">The address of location1 is a null pointer.</exception>
+        [Intrinsic]
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        public static extern short Exchange(ref short location1, short value);
+
         /// <summary>Sets a 32-bit signed integer to a specified value and returns the original value, as an atomic operation.</summary>
         /// <param name="location1">The variable to set to the specified value.</param>
         /// <param name="value">The value to which the <paramref name="location1"/> parameter is set.</param>
@@ -79,13 +97,34 @@ public static partial class Interlocked
         /// <returns>The original value of <paramref name="location1"/>.</returns>
         /// <exception cref="NullReferenceException">The address of location1 is a null pointer.</exception>
         /// <typeparam name="T">The type to be used for <paramref name="location1"/> and <paramref name="value"/>. This type must be a reference type.</typeparam>
+        [Intrinsic]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         [return: NotNullIfNotNull(nameof(location1))]
         public static T Exchange<T>([NotNullIfNotNull(nameof(value))] ref T location1, T value) where T : class? =>
             Unsafe.As<T>(Exchange(ref Unsafe.As<T, object?>(ref location1), value));
         #endregion
 
         #region CompareExchange
+        /// <summary>Compares two 8-bit unsigned integers for equality and, if they are equal, replaces the first value.</summary>
+        /// <param name="location1">The destination, whose value is compared with <paramref name="comparand"/> and possibly replaced.</param>
+        /// <param name="value">The value that replaces the destination value if the comparison results in equality.</param>
+        /// <param name="comparand">The value that is compared to the value at <paramref name="location1"/>.</param>
+        /// <returns>The original value in <paramref name="location1"/>.</returns>
+        /// <exception cref="NullReferenceException">The address of <paramref name="location1"/> is a null pointer.</exception>
+        [Intrinsic]
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        public static extern byte CompareExchange(ref byte location1, byte value, byte comparand);
+
+        /// <summary>Compares two 16-bit signed integers for equality and, if they are equal, replaces the first value.</summary>
+        /// <param name="location1">The destination, whose value is compared with <paramref name="comparand"/> and possibly replaced.</param>
+        /// <param name="value">The value that replaces the destination value if the comparison results in equality.</param>
+        /// <param name="comparand">The value that is compared to the value at <paramref name="location1"/>.</param>
+        /// <returns>The original value in <paramref name="location1"/>.</returns>
+        /// <exception cref="NullReferenceException">The address of <paramref name="location1"/> is a null pointer.</exception>
+        [Intrinsic]
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        public static extern short CompareExchange(ref short location1, short value, short comparand);
+
         /// <summary>Compares two 32-bit signed integers for equality and, if they are equal, replaces the first value.</summary>
         /// <param name="location1">The destination, whose value is compared with <paramref name="comparand"/> and possibly replaced.</param>
         /// <param name="value">The value that replaces the destination value if the comparison results in equality.</param>
@@ -134,8 +173,8 @@ public static partial class Interlocked
         /// <returns>The original value in <paramref name="location1"/>.</returns>
         /// <exception cref="NullReferenceException">The address of <paramref name="location1"/> is a null pointer.</exception>
         /// <typeparam name="T">The type to be used for <paramref name="location1"/>, <paramref name="value"/>, and <paramref name="comparand"/>. This type must be a reference type.</typeparam>
-        [return: NotNullIfNotNull(nameof(location1))]
         [Intrinsic]
+        [return: NotNullIfNotNull(nameof(location1))]
         public static T CompareExchange<T>(ref T location1, T value, T comparand) where T : class? =>
             Unsafe.As<T>(CompareExchange(ref Unsafe.As<T, object?>(ref location1), value, comparand));
         #endregion

src/coreclr/inc/winwrap.h

@@ -264,6 +264,10 @@ WszCreateProcess(
 
 #define InterlockedIncrement            _InterlockedIncrement
 #define InterlockedDecrement            _InterlockedDecrement
+#define InterlockedExchange8            _InterlockedExchange8
+#define InterlockedCompareExchange8     _InterlockedCompareExchange8
+#define InterlockedExchange16           _InterlockedExchange16
+#define InterlockedCompareExchange16    _InterlockedCompareExchange16
 #define InterlockedExchange             _InterlockedExchange
 #define InterlockedCompareExchange      _InterlockedCompareExchange
 #define InterlockedExchangeAdd          _InterlockedExchangeAdd

src/coreclr/jit/codegenxarch.cpp

@@ -4365,7 +4365,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* node)
 
     assert(addr->isUsedFromReg());
     assert(data->isUsedFromReg());
-    assert((size == EA_4BYTE) || (size == EA_PTRSIZE) || (size == EA_GCREF));
+    assert((size <= EA_PTRSIZE) || (size == EA_GCREF));
 
     genConsumeOperands(node);
 

src/coreclr/jit/importercalls.cpp

@@ -3251,16 +3251,23 @@ GenTree* Compiler::impIntrinsic(GenTree*                newobjThis,
             case NI_System_Threading_Interlocked_CompareExchange:
             {
                 var_types retType = JITtype2varType(sig->retType);
-                if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
+                if (TARGET_POINTER_SIZE < genTypeSize(retType))
                 {
                     break;
                 }
+#if defined(TARGET_RISCV64)
+                else if (4 > genTypeSize(retType))
+                {
+                    break;
+                }
+#endif
+
                 if ((retType == TYP_REF) && impStackTop(1).val->IsIntegralConst(0))
                 {
                     // Intrinsify "object" overload in case of null assignment
                     // since we don't need the write barrier.
                 }
-                else if ((retType != TYP_INT) && (retType != TYP_LONG))
+                else if (!varTypeIsIntegral(retType))
                 {
                     break;
                 }
@@ -3286,17 +3293,24 @@ GenTree* Compiler::impIntrinsic(GenTree*                newobjThis,
                 assert(sig->numArgs == 2);
 
                 var_types retType = JITtype2varType(sig->retType);
-                if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
+                if (TARGET_POINTER_SIZE < genTypeSize(retType))
                 {
                     break;
                 }
+#if defined(TARGET_RISCV64)
+                else if (4 > genTypeSize(retType))
+                {
+                    break;
+                }
+#endif
+
                 if ((retType == TYP_REF) && impStackTop().val->IsIntegralConst(0))
                 {
                     // Intrinsify "object" overload in case of null assignment
                     // since we don't need the write barrier.
                     assert(ni == NI_System_Threading_Interlocked_Exchange);
                 }
-                else if ((retType != TYP_INT) && (retType != TYP_LONG))
+                else if (!varTypeIsIntegral(retType))
                 {
                     break;
                 }

src/coreclr/nativeaot/Runtime/EHHelpers.cpp

@@ -283,6 +283,8 @@ EXTERN_C void * RhpAssignRefAVLocation;
 EXTERN_C void * RhpCheckedAssignRefAVLocation;
 EXTERN_C void * RhpCheckedLockCmpXchgAVLocation;
 EXTERN_C void * RhpCheckedXchgAVLocation;
+EXTERN_C void * RhpLockCmpXchg8AVLocation;
+EXTERN_C void * RhpLockCmpXchg16AVLocation;
 EXTERN_C void * RhpLockCmpXchg32AVLocation;
 EXTERN_C void * RhpLockCmpXchg64AVLocation;
 EXTERN_C void * RhpByRefAssignRefAVLocation1;
@@ -305,6 +307,8 @@ static bool InWriteBarrierHelper(uintptr_t faultingIP)
         (uintptr_t)&RhpCheckedAssignRefAVLocation,
         (uintptr_t)&RhpCheckedLockCmpXchgAVLocation,
         (uintptr_t)&RhpCheckedXchgAVLocation,
+        (uintptr_t)&RhpLockCmpXchg8AVLocation,
+        (uintptr_t)&RhpLockCmpXchg16AVLocation,
         (uintptr_t)&RhpLockCmpXchg32AVLocation,
         (uintptr_t)&RhpLockCmpXchg64AVLocation,
         (uintptr_t)&RhpByRefAssignRefAVLocation1,

src/coreclr/nativeaot/Runtime/portable.cpp

@@ -368,6 +368,18 @@ COOP_PINVOKE_HELPER(Object *, RhpCheckedXchg, (Object ** location, Object * valu
     return ret;
 }
 
+COOP_PINVOKE_HELPER(uint8_t, RhpLockCmpXchg8, (uint8_t * location, uint8_t value, uint8_t comparand))
+{
+    // @TODO: USE_PORTABLE_HELPERS - Null check
+    return PalInterlockedCompareExchange(location, value, comparand);
+}
+
+COOP_PINVOKE_HELPER(int16_t, RhpLockCmpXchg16, (int16_t * location, int16_t value, int16_t comparand))
+{
+    // @TODO: USE_PORTABLE_HELPERS - Null check
+    return PalInterlockedCompareExchange(location, value, comparand);
+}
+
 COOP_PINVOKE_HELPER(int32_t, RhpLockCmpXchg32, (int32_t * location, int32_t value, int32_t comparand))
 {
     // @TODO: USE_PORTABLE_HELPERS - Null check

src/coreclr/nativeaot/Runtime/unix/PalRedhawkInline.h

@@ -25,6 +25,24 @@ FORCEINLINE uint32_t PalInterlockedAnd(_Inout_ uint32_t volatile *pDst, uint32_t
     return __sync_and_and_fetch(pDst, iValue);
 }
 
+FORCEINLINE uint8_t PalInterlockedExchange8(_Inout_ uint8_t volatile *pDst, uint8_t iValue)
+{
+#ifdef __clang__
+    return __sync_swap(pDst, iValue);
+#else
+    return __atomic_exchange_n(pDst, iValue, __ATOMIC_ACQ_REL);
+#endif
+}
+
+FORCEINLINE int16_t PalInterlockedExchange16(_Inout_ int16_t volatile *pDst, int16_t iValue)
+{
+#ifdef __clang__
+    return __sync_swap(pDst, iValue);
+#else
+    return __atomic_exchange_n(pDst, iValue, __ATOMIC_ACQ_REL);
+#endif
+}
+
 FORCEINLINE int32_t PalInterlockedExchange(_Inout_ int32_t volatile *pDst, int32_t iValue)
 {
 #ifdef __clang__
@@ -43,6 +61,16 @@ FORCEINLINE int64_t PalInterlockedExchange64(_Inout_ int64_t volatile *pDst, int
 #endif
 }
 
+FORCEINLINE uint8_t PalInterlockedCompareExchange8(_Inout_ uint8_t volatile *pDst, uint8_t iValue, uint8_t iComparand)
+{
+    return __sync_val_compare_and_swap(pDst, iComparand, iValue);
+}
+
+FORCEINLINE int16_t PalInterlockedCompareExchange16(_Inout_ int16_t volatile *pDst, int16_t iValue, int16_t iComparand)
+{
+    return __sync_val_compare_and_swap(pDst, iComparand, iValue);
+}
+
 FORCEINLINE int32_t PalInterlockedCompareExchange(_Inout_ int32_t volatile *pDst, int32_t iValue, int32_t iComparand)
 {
     return __sync_val_compare_and_swap(pDst, iComparand, iValue);

src/coreclr/nativeaot/Runtime/windows/PalRedhawkInline.h

@@ -35,13 +35,41 @@ FORCEINLINE uint32_t PalInterlockedAnd(_Inout_ uint32_t volatile *pDst, uint32_t
     return _InterlockedAnd((long volatile *)pDst, iValue);
 }
 
+EXTERN_C unsigned char __PN__MACHINECALL_CDECL_OR_DEFAULT _InterlockedExchange8(unsigned char volatile *, unsigned char);
+#pragma intrinsic(_InterlockedExchange)
+FORCEINLINE uint8_t PalInterlockedExchange8(_Inout_ uint8_t volatile *pDst, uint8_t iValue)
+{
+    return _InterlockedExchange8((unsigned char volatile *)pDst, iValue);
+}
+
+EXTERN_C short __PN__MACHINECALL_CDECL_OR_DEFAULT _InterlockedExchange16(short volatile *, short);
+#pragma intrinsic(_InterlockedExchange)
+FORCEINLINE int16_t PalInterlockedExchange16(_Inout_ int16_t volatile *pDst, int16_t iValue)
+{
+    return _InterlockedExchange16((short volatile *)pDst, iValue);
+}
+
 EXTERN_C long __PN__MACHINECALL_CDECL_OR_DEFAULT _InterlockedExchange(long volatile *, long);
 #pragma intrinsic(_InterlockedExchange)
 FORCEINLINE int32_t PalInterlockedExchange(_Inout_ int32_t volatile *pDst, int32_t iValue)
 {
     return _InterlockedExchange((long volatile *)pDst, iValue);
 }
 
+EXTERN_C unsigned char __PN__MACHINECALL_CDECL_OR_DEFAULT _InterlockedCompareExchange8(unsigned char volatile *, unsigned char, unsigned char);
+#pragma intrinsic(_InterlockedCompareExchange)
+FORCEINLINE uint8_t PalInterlockedCompareExchange8(_Inout_ uint8_t volatile *pDst, uint8_t iValue, uint8_t iComparand)
+{
+    return _InterlockedCompareExchange8((unsigned char volatile *)pDst, iValue, iComparand);
+}
+
+EXTERN_C short __PN__MACHINECALL_CDECL_OR_DEFAULT _InterlockedCompareExchange16(short volatile *, short, short);
+#pragma intrinsic(_InterlockedCompareExchange)
+FORCEINLINE int16_t PalInterlockedCompareExchange16(_Inout_ int16_t volatile *pDst, int16_t iValue, int16_t iComparand)
+{
+    return _InterlockedCompareExchange16((short volatile *)pDst, iValue, iComparand);
+}
+
 EXTERN_C long __PN__MACHINECALL_CDECL_OR_DEFAULT _InterlockedCompareExchange(long volatile *, long, long);
 #pragma intrinsic(_InterlockedCompareExchange)
 FORCEINLINE int32_t PalInterlockedCompareExchange(_Inout_ int32_t volatile *pDst, int32_t iValue, int32_t iComparand)

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs

@@ -827,6 +827,14 @@ ushort ClrInstanceID
         //
         // Interlocked helpers
         //
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        [RuntimeImport(RuntimeLibrary, "RhpLockCmpXchg8")]
+        internal static extern byte InterlockedCompareExchange(ref byte location1, byte value, byte comparand);
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        [RuntimeImport(RuntimeLibrary, "RhpLockCmpXchg16")]
+        internal static extern short InterlockedCompareExchange(ref short location1, short value, short comparand);
+
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         [RuntimeImport(RuntimeLibrary, "RhpLockCmpXchg32")]
         internal static extern int InterlockedCompareExchange(ref int location1, int value, int comparand);

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs

@@ -11,6 +11,18 @@ public static partial class Interlocked
     {
         #region CompareExchange
 
+        [Intrinsic]
+        public static byte CompareExchange(ref byte location1, byte value, byte comparand)
+        {
+            return RuntimeImports.InterlockedCompareExchange(ref location1, value, comparand);
+        }
+
+        [Intrinsic]
+        public static short CompareExchange(ref short location1, short value, short comparand)
+        {
+            return RuntimeImports.InterlockedCompareExchange(ref location1, value, comparand);
+        }
+
         [Intrinsic]
         public static int CompareExchange(ref int location1, int value, int comparand)
         {
@@ -42,6 +54,32 @@ public static long CompareExchange(ref long location1, long value, long comparan
 
         #region Exchange
 
+        [Intrinsic]
+        public static byte Exchange(ref byte location1, byte value)
+        {
+            byte oldValue;
+
+            do
+            {
+                oldValue = location1;
+            } while (CompareExchange(ref location1, value, oldValue) != oldValue);
+
+            return oldValue;
+        }
+
+        [Intrinsic]
+        public static short Exchange(ref short location1, short value)
+        {
+            short oldValue;
+
+            do
+            {
+                oldValue = location1;
+            } while (CompareExchange(ref location1, value, oldValue) != oldValue);
+
+            return oldValue;
+        }
+
         [Intrinsic]
         public static int Exchange(ref int location1, int value)
         {

src/coreclr/pal/inc/pal.h

@@ -3646,6 +3646,20 @@ Return Values
 The function returns the initial value pointed to by Target.
 
 --*/
+Define_InterlockMethod(
+    BYTE,
+    InterlockedExchange8(IN OUT BYTE volatile *Target, BYTE Value),
+    InterlockedExchange8(Target, Value),
+    __atomic_exchange_n(Target, Value, __ATOMIC_ACQ_REL)
+)
+
+Define_InterlockMethod(
+    SHORT,
+    InterlockedExchange16(IN OUT SHORT volatile *Target, SHORT Value),
+    InterlockedExchange16(Target, Value),
+    __atomic_exchange_n(Target, Value, __ATOMIC_ACQ_REL)
+)
+
 Define_InterlockMethod(
     LONG,
     InterlockedExchange(IN OUT LONG volatile *Target, LONG Value),
@@ -3683,6 +3697,26 @@ Return Values
 The return value is the initial value of the destination.
 
 --*/
+Define_InterlockMethod(
+    LONG,
+    InterlockedCompareExchange8(IN OUT BYTE volatile *Destination, IN BYTE Exchange, IN BYTE Comperand),
+    InterlockedCompareExchange8(Destination, Exchange, Comperand),
+    __sync_val_compare_and_swap(
+        Destination, /* The pointer to a variable whose value is to be compared with. */
+        Comperand, /* The value to be compared */
+        Exchange /* The value to be stored */)
+)
+
+Define_InterlockMethod(
+    LONG,
+    InterlockedCompareExchange16(IN OUT SHORT volatile *Destination, IN SHORT Exchange, IN SHORT Comperand),
+    InterlockedCompareExchange16(Destination, Exchange, Comperand),
+    __sync_val_compare_and_swap(
+        Destination, /* The pointer to a variable whose value is to be compared with. */
+        Comperand, /* The value to be compared */
+        Exchange /* The value to be stored */)
+)
+
 Define_InterlockMethod(
     LONG,
     InterlockedCompareExchange(IN OUT LONG volatile *Destination, IN LONG Exchange, IN LONG Comperand),