Skip to content

Commit 13d4e6d

Browse files
tannergoodingtannergooding
authored
Mark and expose additional Vector functions as Intrinsic (#77562)
* Mark and expose additional Vector functions as Intrinsic * Applying formatting patch * Ensure Vector64.CreateScalarUnsafe for long/ulong/double is handled * Ensure we use the right Count * Ensure small types don't sign extend up to int * Ensure reflection invocation is correct for Vector<T> bitwise ops * Adding tests and fixing a couple small typing issues * Fixing the name of the One property * Assert GT_RSH for simd isn't TYP_LONG on xarch * Ensure the correct operation is picked for vector ShiftRightArithmetic * Fixing StoreAlignedNonTemporal on Arm64
1 parent a27ecc5 commit 13d4e6d

27 files changed

Lines changed: 3985 additions & 1445 deletions

‎src/coreclr/jit/compiler.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2350,7 +2350,7 @@ class Compiler
23502350

23512351
GenTree* gtNewZeroConNode(var_types type);
23522352

2353-
GenTree* gtNewOneConNode(var_types type);
2353+
GenTree* gtNewOneConNode(var_types type, var_types simdBaseType = TYP_UNDEF);
23542354

23552355
GenTreeLclVar* gtNewStoreLclVar(unsigned dstLclNum, GenTree* src);
23562356

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 125 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7189,9 +7189,11 @@ GenTree* Compiler::gtNewZeroConNode(var_types type)
71897189
case TYP_SIMD12:
71907190
case TYP_SIMD16:
71917191
case TYP_SIMD32:
7192+
{
71927193
zero = gtNewVconNode(type);
71937194
zero->AsVecCon()->gtSimd32Val = {};
71947195
break;
7196+
}
71957197
#endif // FEATURE_SIMD
71967198

71977199
default:
@@ -7201,9 +7203,10 @@ GenTree* Compiler::gtNewZeroConNode(var_types type)
72017203
return zero;
72027204
}
72037205

7204-
GenTree* Compiler::gtNewOneConNode(var_types type)
7206+
GenTree* Compiler::gtNewOneConNode(var_types type, var_types simdBaseType /* = TYP_UNDEF */)
72057207
{
72067208
GenTree* one;
7209+
72077210
switch (type)
72087211
{
72097212
case TYP_INT:
@@ -7221,6 +7224,88 @@ GenTree* Compiler::gtNewOneConNode(var_types type)
72217224
one = gtNewDconNode(1.0, type);
72227225
break;
72237226

7227+
#ifdef FEATURE_SIMD
7228+
case TYP_SIMD8:
7229+
case TYP_SIMD12:
7230+
case TYP_SIMD16:
7231+
case TYP_SIMD32:
7232+
{
7233+
GenTreeVecCon* vecCon = gtNewVconNode(type);
7234+
7235+
unsigned simdSize = genTypeSize(type);
7236+
uint32_t simdLength = getSIMDVectorLength(simdSize, simdBaseType);
7237+
7238+
switch (simdBaseType)
7239+
{
7240+
case TYP_BYTE:
7241+
case TYP_UBYTE:
7242+
{
7243+
for (uint32_t index = 0; index < simdLength; index++)
7244+
{
7245+
vecCon->gtSimd32Val.u8[index] = 1;
7246+
}
7247+
break;
7248+
}
7249+
7250+
case TYP_SHORT:
7251+
case TYP_USHORT:
7252+
{
7253+
for (uint32_t index = 0; index < simdLength; index++)
7254+
{
7255+
vecCon->gtSimd32Val.u16[index] = 1;
7256+
}
7257+
break;
7258+
}
7259+
7260+
case TYP_INT:
7261+
case TYP_UINT:
7262+
{
7263+
for (uint32_t index = 0; index < simdLength; index++)
7264+
{
7265+
vecCon->gtSimd32Val.u32[index] = 1;
7266+
}
7267+
break;
7268+
}
7269+
7270+
case TYP_LONG:
7271+
case TYP_ULONG:
7272+
{
7273+
for (uint32_t index = 0; index < simdLength; index++)
7274+
{
7275+
vecCon->gtSimd32Val.u64[index] = 1;
7276+
}
7277+
break;
7278+
}
7279+
7280+
case TYP_FLOAT:
7281+
{
7282+
for (uint32_t index = 0; index < simdLength; index++)
7283+
{
7284+
vecCon->gtSimd32Val.f32[index] = 1.0f;
7285+
}
7286+
break;
7287+
}
7288+
7289+
case TYP_DOUBLE:
7290+
{
7291+
for (uint32_t index = 0; index < simdLength; index++)
7292+
{
7293+
vecCon->gtSimd32Val.f64[index] = 1.0;
7294+
}
7295+
break;
7296+
}
7297+
7298+
default:
7299+
{
7300+
unreached();
7301+
}
7302+
}
7303+
7304+
one = vecCon;
7305+
break;
7306+
}
7307+
#endif // FEATURE_SIMD
7308+
72247309
default:
72257310
unreached();
72267311
}
@@ -19224,6 +19309,11 @@ GenTree* Compiler::gtNewSimdBinOpNode(genTreeOps op,
1922419309
// TODO-XARCH-CQ: We could support division by constant for integral types
1922519310
assert(varTypeIsFloating(simdBaseType));
1922619311

19312+
if (varTypeIsArithmetic(op2))
19313+
{
19314+
op2 = gtNewSimdCreateBroadcastNode(type, op2, simdBaseJitType, simdSize, isSimdAsHWIntrinsic);
19315+
}
19316+
1922719317
if (simdSize == 32)
1922819318
{
1922919319
assert(compIsaSupportedDebugOnly(InstructionSet_AVX));
@@ -19244,9 +19334,22 @@ GenTree* Compiler::gtNewSimdBinOpNode(genTreeOps op,
1924419334
case GT_RSH:
1924519335
case GT_RSZ:
1924619336
{
19337+
// float and double don't have actual instructions for shifting
19338+
// so we'll just use the equivalent integer instruction instead.
19339+
19340+
if (simdBaseType == TYP_FLOAT)
19341+
{
19342+
simdBaseJitType = CORINFO_TYPE_INT;
19343+
simdBaseType = TYP_INT;
19344+
}
19345+
else if (simdBaseType == TYP_DOUBLE)
19346+
{
19347+
simdBaseJitType = CORINFO_TYPE_LONG;
19348+
simdBaseType = TYP_LONG;
19349+
}
19350+
1924719351
assert(!varTypeIsByte(simdBaseType));
19248-
assert(!varTypeIsFloating(simdBaseType));
19249-
assert((op != GT_RSH) || !varTypeIsUnsigned(simdBaseType));
19352+
assert((op != GT_RSH) || (!varTypeIsUnsigned(simdBaseType) && !varTypeIsLong(simdBaseType)));
1925019353

1925119354
// "over shifting" is platform specific behavior. We will match the C# behavior
1925219355
// this requires we mask with (sizeof(T) * 8) - 1 which ensures the shift cannot
@@ -19558,6 +19661,11 @@ GenTree* Compiler::gtNewSimdBinOpNode(genTreeOps op,
1955819661
// TODO-AARCH-CQ: We could support division by constant for integral types
1955919662
assert(varTypeIsFloating(simdBaseType));
1956019663

19664+
if (varTypeIsArithmetic(op2))
19665+
{
19666+
op2 = gtNewSimdCreateBroadcastNode(type, op2, simdBaseJitType, simdSize, isSimdAsHWIntrinsic);
19667+
}
19668+
1956119669
if ((simdSize == 8) && (simdBaseType == TYP_DOUBLE))
1956219670
{
1956319671
intrinsic = NI_AdvSimd_DivideScalar;
@@ -19573,9 +19681,22 @@ GenTree* Compiler::gtNewSimdBinOpNode(genTreeOps op,
1957319681
case GT_RSH:
1957419682
case GT_RSZ:
1957519683
{
19576-
assert(!varTypeIsFloating(simdBaseType));
1957719684
assert((op != GT_RSH) || !varTypeIsUnsigned(simdBaseType));
1957819685

19686+
// float and double don't have actual instructions for shifting
19687+
// so we'll just use the equivalent integer instruction instead.
19688+
19689+
if (simdBaseType == TYP_FLOAT)
19690+
{
19691+
simdBaseJitType = CORINFO_TYPE_INT;
19692+
simdBaseType = TYP_INT;
19693+
}
19694+
else if (simdBaseType == TYP_DOUBLE)
19695+
{
19696+
simdBaseJitType = CORINFO_TYPE_LONG;
19697+
simdBaseType = TYP_LONG;
19698+
}
19699+
1957919700
// "over shifting" is platform specific behavior. We will match the C# behavior
1958019701
// this requires we mask with (sizeof(T) * 8) - 1 which ensures the shift cannot
1958119702
// exceed the number of bits available in `T`. This is roughly equivalent to

‎src/coreclr/jit/hwintrinsicarm64.cpp‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -545,8 +545,16 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
545545
break;
546546
}
547547

548-
case NI_Vector64_Create:
549548
case NI_Vector64_CreateScalarUnsafe:
549+
{
550+
if (genTypeSize(simdBaseType) == 8)
551+
{
552+
intrinsic = NI_Vector64_Create;
553+
}
554+
FALLTHROUGH;
555+
}
556+
557+
case NI_Vector64_Create:
550558
case NI_Vector128_Create:
551559
case NI_Vector128_CreateScalarUnsafe:
552560
{
@@ -1041,6 +1049,14 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
10411049
break;
10421050
}
10431051

1052+
case NI_Vector64_get_One:
1053+
case NI_Vector128_get_One:
1054+
{
1055+
assert(sig->numArgs == 0);
1056+
retNode = gtNewOneConNode(retType, simdBaseType);
1057+
break;
1058+
}
1059+
10441060
case NI_Vector64_get_Zero:
10451061
case NI_Vector128_get_Zero:
10461062
{
@@ -1544,11 +1560,12 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
15441560
case NI_Vector128_ShiftRightArithmetic:
15451561
{
15461562
assert(sig->numArgs == 2);
1563+
genTreeOps op = varTypeIsUnsigned(simdBaseType) ? GT_RSZ : GT_RSH;
15471564

15481565
op2 = impPopStack().val;
15491566
op1 = impSIMDPopStack(retType);
15501567

1551-
retNode = gtNewSimdBinOpNode(GT_RSH, retType, op1, op2, simdBaseJitType, simdSize,
1568+
retNode = gtNewSimdBinOpNode(op, retType, op1, op2, simdBaseJitType, simdSize,
15521569
/* isSimdAsHWIntrinsic */ false);
15531570
break;
15541571
}
@@ -1743,17 +1760,18 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
17431760
{
17441761
assert(numArgs == 3);
17451762
GenTree* indexOp = impStackTop(1).val;
1763+
17461764
if (!indexOp->OperIsConst())
17471765
{
1748-
// TODO-XARCH-CQ: We should always import these like we do with GetElement
1766+
// TODO-ARM64-CQ: We should always import these like we do with GetElement
17491767
// If index is not constant use software fallback.
17501768
return nullptr;
17511769
}
17521770

17531771
ssize_t imm8 = indexOp->AsIntCon()->IconValue();
17541772
ssize_t count = simdSize / genTypeSize(simdBaseType);
17551773

1756-
if (imm8 >= count || imm8 < 0)
1774+
if ((imm8 >= count) || (imm8 < 0))
17571775
{
17581776
// Using software fallback if index is out of range (throw exception)
17591777
return nullptr;

0 commit comments

Comments
 (0)