diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 2098d0658..c42f09d56 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -1093,6 +1093,9 @@ public: void AVX128_VCVTPH2PS(OpcodeArgs); void AVX128_VCVTPS2PH(OpcodeArgs); + void AVX128_VBCSTNEBF162PS(OpcodeArgs); + void AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even); + void AVX128_VCVTNEPS2BF16(OpcodeArgs); // End of AVX 128-bit implementation diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp index 0079f632f..a219891c6 100644 --- a/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp +++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp @@ -2355,4 +2355,75 @@ void OpDispatchBuilder::AVX128_VCVTPS2PH(OpcodeArgs) { } } +void OpDispatchBuilder::AVX128_VBCSTNEBF162PS(OpcodeArgs) { + const auto DstSize = GetDstSize(Op); + const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE; + + auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags); + + // bfloat16 is converted to fp32 and broadcast through the vector. + // bfloat16->float32 is left-shift by 16. + Src1 = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src1, 16); + Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0); + + RefPair Result {}; + Result.Low = Src1; + + if (Is128Bit) { + Result.High = LoadZeroVector(OpSize::i128Bit); + } else { + Result.High = Result.Low; + } + AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result); +} + +void OpDispatchBuilder::AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even) { + const auto DstSize = GetDstSize(Op); + const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE; + + auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit); + + RefPair Result {}; + + auto Convert = [&](auto Src, bool Even) -> Ref { + // Even or odd elements of each 128-bit register is converted from bfloat16 to float32. + // For even, just left shift the elements. + if (Even) { + return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src, 16); + } + + // For Odds, we just need to mask off the lower 16-bits of each 32-bit element. + // Easiest way to do this is with a right then left shift. + auto Tmp = _VUShrI(OpSize::i128Bit, OpSize::i32Bit, Src, 16); + return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Tmp, 16); + }; + + Result.Low = Convert(Src.Low, Even); + + if (Is128Bit) { + Result.High = LoadZeroVector(OpSize::i128Bit); + } else { + Result.High = Convert(Src.High, Even); + } + AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result); +} + +void OpDispatchBuilder::AVX128_VCVTNEPS2BF16(OpcodeArgs) { + const auto SrcSize = OpSizeFromSrc(Op); + const auto Is128BitSrc = SrcSize == OpSize::i128Bit; + auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc); + + // Convert 4 or 8 32-bits floats in to bfloat16. + // This instruction only operates with rounding mode set to `Rounds to Nearest (even)`. + + RefPair Result {}; + Result.Low = _VBFCVTN(OpSize::i128Bit, Src.Low); + if (!Is128BitSrc) { + Result.Low = _VBFCVTN2(OpSize::i128Bit, Result.Low, Src.High); + } + + Result.High = LoadZeroVector(OpSize::i128Bit); + AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result); +} + } // namespace FEXCore::IR diff --git a/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp b/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp index 85b2aac04..a483c4fbe 100644 --- a/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp +++ b/FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp @@ -321,6 +321,7 @@ namespace AVX128 { {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i64Bit>}, {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i128Bit>}, + {OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVX128_VCVTNEPS2BF16}, {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i8Bit>}, {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i16Bit>}, @@ -365,6 +366,13 @@ namespace AVX128 { {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD + {OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, false>}, + {OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, true>}, + {OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, true>}, + {OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, false>}, + {OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNEBF162PS}, + {OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNEBF162PS}, + {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD @@ -771,6 +779,7 @@ namespace AVX256 { {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>}, {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>}, + {OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::UnimplementedOp}, {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i8Bit>}, {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i16Bit>}, @@ -815,6 +824,13 @@ namespace AVX256 { {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 1, 3>}, // VFMADDSUB {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD + {OPD(2, 0b00, 0xB0), 1, &IR::OpDispatchBuilder::UnimplementedOp}, + {OPD(2, 0b01, 0xB0), 1, &IR::OpDispatchBuilder::UnimplementedOp}, + {OPD(2, 0b10, 0xB0), 1, &IR::OpDispatchBuilder::UnimplementedOp}, + {OPD(2, 0b11, 0xB0), 1, &IR::OpDispatchBuilder::UnimplementedOp}, + {OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::UnimplementedOp}, + {OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::UnimplementedOp}, + {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>}, // VFMADDSUB {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD