Skip to content

Commit 1bb6344

Browse files
saucecontroltannergooding
authored and
iremyux
committed
JIT: Move remaining xarch floating->integral cast implementation to lowering (dotnet#117571)
This changes the lowering of floating->integral casts to always replace the cast node with HWIntrinsics rather than doing fixups ahead of the cast and leaving the node in place as a self-cast or letting it be handled in codegen. Since the self-cast was not always eliminated in codegen, this results in some size and throughput improvements. Because the cast is always replaced now, `genFloatToIntCast` is no longer necessary on xarch. This is best viewed with whitespace ignored. Most of the changes are simply an extra level of indentation for the pre-AVX10.2 code in `LowerCast`. [Diffs](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1235886&view=ms.vss-build-web.run-extensions-tab) --------- Co-authored-by: Tanner Gooding <tagoo@microsoft.com>
1 parent d9243ee commit 1bb6344

10 files changed

Lines changed: 326 additions & 420 deletions

‎src/coreclr/jit/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ function(create_standalone_jit)
7575
endif ()
7676
endfunction()
7777

78-
if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64 OR (CLR_CMAKE_TARGET_ARCH_I386 AND NOT CLR_CMAKE_HOST_UNIX))
78+
if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGET_ARCH_I386)
7979
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:FEATURE_SIMD>)
8080
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:FEATURE_HW_INTRINSICS>)
8181
add_compile_definitions($<$<NOT:$<BOOL:$<TARGET_PROPERTY:IGNORE_DEFAULT_TARGET_ARCH>>>:FEATURE_MASKED_HW_INTRINSICS>)

‎src/coreclr/jit/codegenlinear.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2330,8 +2330,13 @@ void CodeGen::genCodeForCast(GenTreeOp* tree)
23302330
}
23312331
else if (varTypeIsFloating(tree->gtOp1))
23322332
{
2333+
#ifdef TARGET_XARCH
2334+
// These casts should have been lowered to HWIntrinsics
2335+
unreached();
2336+
#else
23332337
// Casts float/double --> int32/int64
23342338
genFloatToIntCast(tree);
2339+
#endif
23352340
}
23362341
else if (varTypeIsFloating(targetType))
23372342
{

‎src/coreclr/jit/codegenxarch.cpp‎

Lines changed: 0 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -7216,74 +7216,6 @@ void CodeGen::genIntToFloatCast(GenTree* treeNode)
72167216
genProduceReg(treeNode);
72177217
}
72187218

7219-
//------------------------------------------------------------------------
7220-
// genFloatToIntCast: Generate code to cast float/double to int/long
7221-
//
7222-
// Arguments:
7223-
// treeNode - The GT_CAST node
7224-
//
7225-
// Return Value:
7226-
// None.
7227-
//
7228-
// Assumptions:
7229-
// Cast is a non-overflow conversion.
7230-
// The treeNode must have an assigned register.
7231-
// SrcType=float/double and DstType= int32/uint32/int64/uint64
7232-
//
7233-
// TODO-XArch-CQ: (Low-pri) - generate in-line code when DstType = uint64
7234-
//
7235-
void CodeGen::genFloatToIntCast(GenTree* treeNode)
7236-
{
7237-
// we don't expect to see overflow detecting float/double --> int type conversions here
7238-
// as they should have been converted into helper calls by front-end.
7239-
assert(treeNode->OperIs(GT_CAST));
7240-
assert(!treeNode->gtOverflow());
7241-
7242-
regNumber targetReg = treeNode->GetRegNum();
7243-
assert(genIsValidIntReg(targetReg));
7244-
7245-
GenTree* op1 = treeNode->AsOp()->gtOp1;
7246-
#ifdef DEBUG
7247-
if (op1->isUsedFromReg())
7248-
{
7249-
assert(genIsValidFloatReg(op1->GetRegNum()));
7250-
}
7251-
#endif
7252-
7253-
var_types dstType = treeNode->CastToType();
7254-
var_types srcType = op1->TypeGet();
7255-
assert(varTypeIsFloating(srcType) && !varTypeIsFloating(dstType));
7256-
7257-
// We should never be seeing dstType whose size is neither sizeof(TYP_INT) nor sizeof(TYP_LONG).
7258-
// For conversions to byte/sbyte/int16/uint16 from float/double, we would expect the
7259-
// front-end or lowering phase to have generated two levels of cast. The first one is
7260-
// for float or double to int32/uint32 and the second one for narrowing int32/uint32 to
7261-
// the required smaller int type.
7262-
emitAttr dstSize = EA_ATTR(genTypeSize(dstType));
7263-
noway_assert((dstSize == EA_ATTR(genTypeSize(TYP_INT))) || (dstSize == EA_ATTR(genTypeSize(TYP_LONG))));
7264-
7265-
// We shouldn't be seeing uint64 here as it should have been converted
7266-
// into a helper call by either front-end or lowering phase, unless we have AVX512
7267-
// accelerated conversions.
7268-
assert(!varTypeIsUnsigned(dstType) || (dstSize != EA_ATTR(genTypeSize(TYP_LONG))) ||
7269-
compiler->canUseEvexEncodingDebugOnly());
7270-
7271-
// If the dstType is TYP_UINT, we have 32-bits to encode the
7272-
// float number. Any of 33rd or above bits can be the sign bit.
7273-
// To achieve it we pretend as if we are converting it to a long.
7274-
if (varTypeIsUnsigned(dstType) && (dstSize == EA_ATTR(genTypeSize(TYP_INT))) && !compiler->canUseEvexEncoding())
7275-
{
7276-
dstType = TYP_LONG;
7277-
}
7278-
7279-
// Note that we need to specify dstType here so that it will determine
7280-
// the size of destination integer register and also the rex.w prefix.
7281-
genConsumeOperands(treeNode->AsOp());
7282-
instruction ins = ins_FloatConv(dstType, srcType);
7283-
GetEmitter()->emitInsBinary(ins, emitTypeSize(dstType), treeNode, op1);
7284-
genProduceReg(treeNode);
7285-
}
7286-
72877219
//------------------------------------------------------------------------
72887220
// genCkfinite: Generate code for ckfinite opcode.
72897221
//

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21953,19 +21953,19 @@ GenTree* Compiler::gtNewSimdCvtNode(
2195321953
switch (simdTargetBaseType)
2195421954
{
2195521955
case TYP_INT:
21956-
cvtIntrinsic = NI_AVX10v2_ConvertToVectorInt32WithTruncationSaturation;
21956+
cvtIntrinsic = NI_AVX10v2_ConvertToVectorInt32WithTruncatedSaturation;
2195721957
break;
2195821958

2195921959
case TYP_UINT:
21960-
cvtIntrinsic = NI_AVX10v2_ConvertToVectorUInt32WithTruncationSaturation;
21960+
cvtIntrinsic = NI_AVX10v2_ConvertToVectorUInt32WithTruncatedSaturation;
2196121961
break;
2196221962

2196321963
case TYP_LONG:
21964-
cvtIntrinsic = NI_AVX10v2_ConvertToVectorInt64WithTruncationSaturation;
21964+
cvtIntrinsic = NI_AVX10v2_ConvertToVectorInt64WithTruncatedSaturation;
2196521965
break;
2196621966

2196721967
case TYP_ULONG:
21968-
cvtIntrinsic = NI_AVX10v2_ConvertToVectorUInt64WithTruncationSaturation;
21968+
cvtIntrinsic = NI_AVX10v2_ConvertToVectorUInt64WithTruncatedSaturation;
2196921969
break;
2197021970

2197121971
default:

‎src/coreclr/jit/hwintrinsic.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
978978
{ NI_Illegal, NI_Illegal }, // AVX512v2_X64
979979
{ NI_Illegal, NI_Illegal }, // AVX512v3_X64
980980
{ NI_Illegal, NI_Illegal }, // AVX10v1_X64
981-
{ NI_Illegal, NI_Illegal }, // AVX10v2_X64
981+
{ FIRST_NI_AVX10v2_X64, LAST_NI_AVX10v2_X64 }, // AVX10v2_X64
982982
{ NI_Illegal, NI_Illegal }, // AES_X64
983983
{ NI_Illegal, NI_Illegal }, // AVX512VP2INTERSECT_X64
984984
{ NI_Illegal, NI_Illegal }, // AVXIFMA_X64

‎src/coreclr/jit/hwintrinsiccodegenxarch.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1011,6 +1011,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
10111011
case InstructionSet_AVX512:
10121012
case InstructionSet_AVX512_X64:
10131013
case InstructionSet_AVX512v2:
1014+
case InstructionSet_AVX10v2:
1015+
case InstructionSet_AVX10v2_X64:
10141016
case InstructionSet_AVXVNNIINT:
10151017
case InstructionSet_AVXVNNIINT_V512:
10161018
{
@@ -3502,6 +3504,10 @@ void CodeGen::genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOption
35023504
case NI_AVX512_X64_ConvertToInt64:
35033505
case NI_AVX512_X64_ConvertToUInt64:
35043506
case NI_AVX512_X64_ConvertToUInt64WithTruncation:
3507+
case NI_AVX10v2_ConvertToInt32WithTruncatedSaturation:
3508+
case NI_AVX10v2_ConvertToUInt32WithTruncatedSaturation:
3509+
case NI_AVX10v2_X64_ConvertToInt64WithTruncatedSaturation:
3510+
case NI_AVX10v2_X64_ConvertToUInt64WithTruncatedSaturation:
35053511
{
35063512
assert(baseType == TYP_DOUBLE || baseType == TYP_FLOAT);
35073513
emitAttr attr = emitTypeSize(targetType);
@@ -3512,7 +3518,6 @@ void CodeGen::genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOption
35123518
}
35133519

35143520
case NI_AVX512_ConvertToVector128UInt32:
3515-
case NI_AVX512_ConvertToVector128UInt32WithSaturation:
35163521
case NI_AVX512_ConvertToVector256Int32:
35173522
case NI_AVX512_ConvertToVector256UInt32:
35183523
{
@@ -3535,6 +3540,7 @@ void CodeGen::genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOption
35353540
case NI_AVX512_ConvertToVector128SByteWithSaturation:
35363541
case NI_AVX512_ConvertToVector128UInt16:
35373542
case NI_AVX512_ConvertToVector128UInt16WithSaturation:
3543+
case NI_AVX512_ConvertToVector128UInt32WithSaturation:
35383544
case NI_AVX512_ConvertToVector256Byte:
35393545
case NI_AVX512_ConvertToVector256ByteWithSaturation:
35403546
case NI_AVX512_ConvertToVector256Int16:

‎src/coreclr/jit/hwintrinsiclistxarch.h‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,19 +1077,31 @@ HARDWARE_INTRINSIC(AVX512v3, ExpandLoad,
10771077
#define FIRST_NI_AVX10v2 NI_AVX10v2_ConvertToByteWithSaturationAndZeroExtendToInt32
10781078
HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithSaturationAndZeroExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2iubs, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible)
10791079
HARDWARE_INTRINSIC(AVX10v2, ConvertToByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2iubs, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1080+
HARDWARE_INTRINSIC(AVX10v2, ConvertToInt32WithTruncatedSaturation, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis32, INS_vcvttsd2sis32}, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
10801081
HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithSaturationAndZeroExtendToInt32, -1, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvtps2ibs, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbRoundingCompatible)
10811082
HARDWARE_INTRINSIC(AVX10v2, ConvertToSByteWithTruncatedSaturationAndZeroExtendToInt32, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2ibs, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1082-
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt32WithTruncationSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2dqs, INS_vcvttpd2dqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1083-
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt64WithTruncationSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qqs, INS_vcvttpd2qqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1084-
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt32WithTruncationSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udqs, INS_vcvttpd2udqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1085-
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt64WithTruncationSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqqs, INS_vcvttpd2uqqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1083+
HARDWARE_INTRINSIC(AVX10v2, ConvertToUInt32WithTruncatedSaturation, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis32, INS_vcvttsd2usis32}, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
1084+
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt32WithTruncatedSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2dqs, INS_vcvttpd2dqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1085+
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorInt64WithTruncatedSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2qqs, INS_vcvttpd2qqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1086+
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt32WithTruncatedSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2udqs, INS_vcvttpd2udqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
1087+
HARDWARE_INTRINSIC(AVX10v2, ConvertToVectorUInt64WithTruncatedSaturation, -1, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttps2uqqs, INS_vcvttpd2uqqs}, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromFirstArg)
10861088
HARDWARE_INTRINSIC(AVX10v2, MinMax, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxps, INS_vminmaxpd}, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg)
10871089
HARDWARE_INTRINSIC(AVX10v2, MinMaxScalar, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vminmaxss, INS_vminmaxsd}, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg)
10881090
HARDWARE_INTRINSIC(AVX10v2, MoveScalar, 16, -1, {INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_vmovd_simd, INS_vmovd_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMDScalar, HW_Flag_NoContainment)
10891091
HARDWARE_INTRINSIC(AVX10v2, MultipleSumAbsoluteDifferences, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_vmpsadbw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_FullRangeIMM)
10901092
HARDWARE_INTRINSIC(AVX10v2, StoreScalar, 16, 2, {INS_invalid, INS_invalid, INS_vmovw_simd, INS_vmovw_simd, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryStore, HW_Flag_NoRMWSemantics|HW_Flag_BaseTypeFromSecondArg)
10911093
#define LAST_NI_AVX10v2 NI_AVX10v2_StoreScalar
10921094

1095+
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
1096+
// ISA Function name SIMD size NumArg Instructions Category Flags
1097+
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}
1098+
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
1099+
// 64-bit only intrinsics for AVX10v2
1100+
#define FIRST_NI_AVX10v2_X64 NI_AVX10v2_X64_ConvertToInt64WithTruncatedSaturation
1101+
HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToInt64WithTruncatedSaturation, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2sis64, INS_vcvttsd2sis64}, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
1102+
HARDWARE_INTRINSIC(AVX10v2_X64, ConvertToUInt64WithTruncatedSaturation, 16, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vcvttss2usis64, INS_vcvttsd2usis64}, HW_Category_SIMDScalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
1103+
#define LAST_NI_AVX10v2_X64 NI_AVX10v2_X64_ConvertToUInt64WithTruncatedSaturation
1104+
10931105
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
10941106
// ISA Function name SIMD size NumArg Instructions Category Flags
10951107
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}

0 commit comments

Comments
 (0)