Fix SIMD complement and scalar FMA folding

src/coreclr/jit/lowerxarch.cpp

@@ -1313,6 +1313,12 @@ void Lowering::LowerFusedMultiplyOp(GenTreeHWIntrinsic* node)
             continue;
         }
 
+        if (isScalar && (i == 1))
+        {
+            // Scalar FMA copies the upper elements of its first operand, including any vector negation.
+            continue;
+        }
+
         if (!arg->OperIsHWIntrinsic())
         {
             continue;

src/coreclr/jit/morph.cpp

@@ -10079,6 +10079,12 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node)
 
             if (GenTree::OperIsCompare(op1Oper))
             {
+                if (op1IsScalar)
+                {
+                    // Reversing a scalar comparison does not complement the upper elements.
+                    break;
+                }
+
                 assert(op1Intrin->GetOperandCount() == 2);
 
                 GenTree* cmpOp1 = op1Intrin->Op(1);
@@ -10087,9 +10093,8 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node)
                 const bool reverseCond = true;
 
                 var_types lookupType =
-                    op1IsScalar ? op1RetType
-                                : GenTreeHWIntrinsic::GetLookupTypeForCmpOp(this, op1Oper, op1RetType, op1SimdBaseType,
-                                                                            op1SimdSize, reverseCond);
+                    GenTreeHWIntrinsic::GetLookupTypeForCmpOp(this, op1Oper, op1RetType, op1SimdBaseType, op1SimdSize,
+                                                              reverseCond);
                 NamedIntrinsic newId =
                     GenTreeHWIntrinsic::GetHWIntrinsicIdForCmpOp(this, op1Oper, lookupType, cmpOp1, cmpOp2,
                                                                  op1SimdBaseType, op1SimdSize, op1IsScalar,
@@ -10129,7 +10134,6 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node)
                 switch (op1Intrinsic)
                 {
                     case NI_AVX_Compare:
-                    case NI_AVX_CompareScalar:
                     case NI_AVX512_CompareMask:
                     {
                         assert(op1Intrin->GetOperandCount() == 3);
@@ -10187,7 +10191,7 @@ GenTree* Compiler::fgOptimizeHWIntrinsic(GenTreeHWIntrinsic* node)
                         if (newMode != mode)
                         {
                             ExtractEffectiveOp(GT_NOT, node, /* destroyNodes */ true);
-                            cmpOp3->AsIntConCommon()->SetIntegralValue(static_cast<uint8_t>(mode));
+                            cmpOp3->AsIntConCommon()->SetIntegralValue(static_cast<uint8_t>(newMode));
                             fgUpdateConstTreeValueNumber(cmpOp3);
                             return fgMorphHWIntrinsicRequired(op1Intrin);
                         }

src/tests/JIT/Regression/JitBlue/Runtime_133518/Runtime_133518.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using Xunit;
+
+public class Runtime_133518
+{
+    [ConditionalTheory(typeof(Avx), nameof(Avx.IsSupported))]
+    [InlineData(1.0, 1.0)]
+    [InlineData(1.0, 2.0)]
+    [InlineData(double.NaN, 1.0)]
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    public static void TestEntryPoint(double left, double right)
+    {
+        bool unordered = double.IsNaN(left) || double.IsNaN(right);
+        int notEqual = !unordered && (left != right) ? -1 : 0;
+        int equal = unordered || (left == right) ? -1 : 0;
+        Vector256<float> a = Vector256.Create((float)left);
+        Vector256<float> b = Vector256.Create((float)right);
+        Vector256<double> c = Vector256.Create(left);
+        Vector256<double> d = Vector256.Create(right);
+
+        Assert.Equal(Vector256.Create(notEqual), (~Avx.Compare(a, b, FloatComparisonMode.UnorderedEqualNonSignaling)).AsInt32());
+        Assert.Equal(Vector256.Create(equal), (~Avx.Compare(a, b, FloatComparisonMode.OrderedNotEqualNonSignaling)).AsInt32());
+        Assert.Equal(Vector256.Create(-1), (~Avx.Compare(a, b, FloatComparisonMode.OrderedFalseNonSignaling)).AsInt32());
+        Assert.Equal(Vector256<int>.Zero, (~Avx.Compare(a, b, FloatComparisonMode.UnorderedTrueNonSignaling)).AsInt32());
+
+        Assert.Equal(Vector256.Create((long)notEqual), (~Avx.Compare(c, d, FloatComparisonMode.UnorderedEqualNonSignaling)).AsInt64());
+        Assert.Equal(Vector256.Create((long)equal), (~Avx.Compare(c, d, FloatComparisonMode.OrderedNotEqualNonSignaling)).AsInt64());
+        Assert.Equal(Vector256.Create(-1L), (~Avx.Compare(c, d, FloatComparisonMode.OrderedFalseNonSignaling)).AsInt64());
+        Assert.Equal(Vector256<long>.Zero, (~Avx.Compare(c, d, FloatComparisonMode.UnorderedTrueNonSignaling)).AsInt64());
+
+        if (Avx512F.IsSupported)
+        {
+            Assert.Equal(Vector512.Create(equal),
+                (~Avx512F.Compare(Vector512.Create((float)left), Vector512.Create((float)right), FloatComparisonMode.OrderedNotEqualNonSignaling)).AsInt32());
+            Assert.Equal(Vector512.Create((long)equal),
+                (~Avx512F.Compare(Vector512.Create(left), Vector512.Create(right), FloatComparisonMode.OrderedNotEqualNonSignaling)).AsInt64());
+        }
+    }
+}

src/tests/JIT/Regression/JitBlue/Runtime_133521/Runtime_133521.cs

@@ -0,0 +1,91 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using System.Runtime.Intrinsics.X86;
+using Xunit;
+
+public class Runtime_133521
+{
+    [ConditionalTheory(typeof(Sse2), nameof(Sse2.IsSupported))]
+    [InlineData(1.0, 1.0)]
+    [InlineData(1.0, 2.0)]
+    [InlineData(double.NaN, 1.0)]
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    public static void ScalarComparison(double left, double right)
+    {
+        Vector128<float> a = Vector128.Create((float)left, 2.0f, -0.0f, float.NaN);
+        Vector128<float> b = Vector128.Create((float)right);
+        Vector128<double> c = Vector128.Create(left, 2.0);
+        Vector128<double> d = Vector128.Create(right);
+        uint expectedSingle = left == right ? 0u : uint.MaxValue;
+        ulong expectedDouble = left == right ? 0ul : ulong.MaxValue;
+        Vector128<uint> expectedScalarSingle = Vector128.Create(expectedSingle, ~0x40000000u, ~0x80000000u, ~BitConverter.SingleToUInt32Bits(float.NaN));
+        Vector128<ulong> expectedScalarDouble = Vector128.Create(expectedDouble, ~0x4000000000000000ul);
+
+        Assert.Equal(expectedScalarSingle, (~Sse.CompareScalarEqual(a, b)).AsUInt32());
+        Assert.Equal(expectedScalarDouble, (~Sse2.CompareScalarEqual(c, d)).AsUInt64());
+
+        if (Avx.IsSupported)
+        {
+            bool unordered = double.IsNaN(left) || double.IsNaN(right);
+            Assert.Equal(expectedScalarSingle.WithElement(0, unordered ? 0u : expectedSingle),
+                (~Avx.CompareScalar(a, b, FloatComparisonMode.UnorderedEqualNonSignaling)).AsUInt32());
+            Assert.Equal(expectedScalarDouble.WithElement(0, unordered ? 0ul : expectedDouble),
+                (~Avx.CompareScalar(c, d, FloatComparisonMode.UnorderedEqualNonSignaling)).AsUInt64());
+        }
+    }
+
+    [ConditionalTheory(typeof(Fma), nameof(Fma.IsSupported))]
+    [InlineData(0, -11)]
+    [InlineData(1, -13)]
+    [InlineData(2, 13)]
+    [InlineData(3, 11)]
+    public static void ScalarFmaNegation(int operation, int expected)
+    {
+        Vector128<float> a = Vector128.Create(2.0f, 3.0f, -0.0f, float.NaN);
+        Vector128<double> b = Vector128.Create(2.0, -0.0);
+        Assert.Equal(Vector128.Create((float)expected, -3.0f, 0.0f, BitConverter.UInt32BitsToSingle(BitConverter.SingleToUInt32Bits(float.NaN) ^ 0x80000000u)).AsUInt32(),
+            NegatedScalarFma(a, Vector128.Create(6.0f), Vector128.Create(1.0f), operation).AsUInt32());
+        Assert.Equal(Vector128.Create((double)expected, 0.0).AsUInt64(),
+            NegatedScalarFma(b, Vector128.Create(6.0), Vector128.Create(1.0), operation).AsUInt64());
+
+        if (operation == 0)
+        {
+            Assert.Equal(Vector128.Create(-13.0f, 3.0f, -0.0f, float.NaN).AsUInt32(),
+                NegatedOtherScalarFma(a, Vector128.Create(6.0f), Vector128.Create(1.0f)).AsUInt32());
+            Assert.Equal(Vector128.Create(-13.0, -0.0).AsUInt64(),
+                NegatedOtherScalarFma(b, Vector128.Create(6.0), Vector128.Create(1.0)).AsUInt64());
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static Vector128<float> NegatedScalarFma(Vector128<float> a, Vector128<float> b, Vector128<float> c, int operation) => operation switch
+    {
+        0 => Fma.MultiplyAddScalar(-a, b, c),
+        1 => Fma.MultiplySubtractScalar(-a, b, c),
+        2 => Fma.MultiplyAddNegatedScalar(-a, b, c),
+        3 => Fma.MultiplySubtractNegatedScalar(-a, b, c),
+        _ => throw new ArgumentOutOfRangeException(nameof(operation)),
+    };
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static Vector128<double> NegatedScalarFma(Vector128<double> a, Vector128<double> b, Vector128<double> c, int operation) => operation switch
+    {
+        0 => Fma.MultiplyAddScalar(-a, b, c),
+        1 => Fma.MultiplySubtractScalar(-a, b, c),
+        2 => Fma.MultiplyAddNegatedScalar(-a, b, c),
+        3 => Fma.MultiplySubtractNegatedScalar(-a, b, c),
+        _ => throw new ArgumentOutOfRangeException(nameof(operation)),
+    };
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static Vector128<float> NegatedOtherScalarFma(Vector128<float> a, Vector128<float> b, Vector128<float> c) =>
+        Fma.MultiplyAddScalar(a, -b, -c);
+
+    [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
+    private static Vector128<double> NegatedOtherScalarFma(Vector128<double> a, Vector128<double> b, Vector128<double> c) =>
+        Fma.MultiplyAddScalar(a, -b, -c);
+}

src/tests/JIT/Regression/Regression_ro_2.csproj

@@ -143,6 +143,8 @@
     <Compile Include="JitBlue\Runtime_133022\Runtime_133022.cs" />
     <Compile Include="JitBlue\Runtime_133101\Runtime_133101.cs" />
     <Compile Include="JitBlue\Runtime_133207\Runtime_133207.cs" />
+    <Compile Include="JitBlue\Runtime_133518\Runtime_133518.cs" />
+    <Compile Include="JitBlue\Runtime_133521\Runtime_133521.cs" />
   </ItemGroup>
   <Import Project="$(TestSourceDir)MergedTestRunner.targets" />
 </Project>