From 0d475734fb7f3b9d5f0c2ce8f6bb4d77ab695d8a Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Thu, 3 Sep 2026 11:16:36 +1200 Subject: [PATCH] [HLSL] Add thread-scope MatVec semantics coverage Thread-scope matrix operations are per-thread, so every lane must load its own matrix and multiply against its own vector. The existing thread-scope cases give every thread identical data, so an implementation that loads one matrix and broadcasts it to the whole wave passes them. Neither case here can pass that way. MatVecMul_Thread_4x8_F16_PerThread gives each of 8 threads a distinct 4x8 matrix in its own 128-byte aligned slot, with element (t,m,c) = Z + c where Z = 4t + m + 1. Vector A is {1..8}, so the expected row result is the closed form 36*Z + 168 over sum(c+1) = 36 and sum(c*(c+1)) = 168. MatVecMul_Thread_4x8_F16_Divergent runs the same data under non-uniform control flow, with each arm multiplying by a different vector. Vector B is all -1, giving -(8*Z + 28) over sum(c) = 28. The two arms are therefore disjoint by sign, so no arm-A result can be confused with an arm-B result anywhere in the output. An earlier revision used all +1, which collided in two places (A(Z=1) = B(Z=22) = 204 and A(Z=3) = B(Z=31) = 276), so a swap of those slots would have been invisible. Because the arm is chosen by lane parity, any thread may legally land on either arm, so the portable bounds are A in [204, 1320] and B in [-284, -36]. The largest single product is 312 and every partial sum is bounded by its own final magnitude, so all inputs, products and results are exactly representable in F16. The divergent case branches on WaveGetLaneIndex() rather than SV_GroupThreadID. Thread parity only guarantees that different threads take different arms; the distribution of threads to waves is implementation defined (hlsl-specs 0048-group-wave-index.md), so an implementation that segregates parities into separate waves would make both arms wave-uniform and the case would prove nothing while still passing. Lane parity guarantees divergence inside the wave. Because that guarantee still depends on the wave containing more than one active thread, each thread records a witness word before the branch holding its predicate bit and a WaveActiveAnyTrue vote in both directions. The host checks that at least one thread reports a wave that executed both arms. If none does, the case reports Skipped rather than Failed. Implementations may launch additional waves, those waves may contain inactive lanes, and the distribution of threads among them is implementation defined, so a wave holding only same-parity lanes is legal. Failing there would reject a conformant implementation for a precondition this test cannot itself guarantee. The per-thread results are still verified first and a wrong value still fails, so the downgrade applies only when nothing else is wrong. Each arm also writes a branch-local marker. The host requires the marker to agree with the pre-branch witness, and validates results against the witness rather than the marker. Validating against the marker would be circular: an implementation that executed the wrong arm would write that arm's marker, and the host would then check it against the oracle for the arm that actually ran instead of the one the predicate selected. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 387 ++++++++++++++++++ 1 file changed, 387 insertions(+) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index 56dbd9b12c..10ec5b39e8 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -3591,6 +3591,8 @@ class DxilConf_SM610_LinAlg { // Matrix Vector Arithmetic TEST_METHOD(MatVecMul_Thread_16x16_F16); TEST_METHOD(MatVecMul_Thread_4x8_F32); + TEST_METHOD(MatVecMul_Thread_4x8_F16_PerThread); + TEST_METHOD(MatVecMul_Thread_4x8_F16_Divergent); TEST_METHOD(MatVecMul_Thread_4x8_F16_NonUniform); TEST_METHOD(MatVecMul_Thread_4x8_F16_ColumnMajor); TEST_METHOD(MatVecMul_Thread_4x8_I8_Interpreted); @@ -7030,6 +7032,391 @@ void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F32() { /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F32); } +// Thread-scope cases that give each thread distinct matrix values, the second +// under divergent control flow. +static constexpr int ThreadSemanticsThreads = 8; +static constexpr int ThreadSemanticsM = 4; +static constexpr int ThreadSemanticsN = 8; +static constexpr size_t ThreadSemanticsElementBytes = sizeof(HLSLHalf_t); +static constexpr size_t ThreadSemanticsStrideBytes = + alignMatrixStride(ThreadSemanticsN * ThreadSemanticsElementBytes); +static constexpr size_t ThreadSemanticsMatrixSlotBytes = + MatrixOffsetAlignmentBytes; +static constexpr size_t ThreadSemanticsResultBytes = + ThreadSemanticsM * ThreadSemanticsElementBytes; +static constexpr size_t ThreadSemanticsWitnessOffset = + ThreadSemanticsResultBytes; +static constexpr size_t ThreadSemanticsArmOffset = + ThreadSemanticsWitnessOffset + sizeof(uint32_t); +static constexpr size_t ThreadSemanticsOutputSlotBytes = + ThreadSemanticsArmOffset + sizeof(uint32_t); +static constexpr size_t ThreadSemanticsVectorBytes = + ThreadSemanticsN * ThreadSemanticsElementBytes; + +// Bits of the witness word each thread stores before the divergent branch. +static constexpr uint32_t ThreadSemanticsWitnessUseB = 1u; +static constexpr uint32_t ThreadSemanticsWitnessMixed = 2u; +static constexpr uint32_t ThreadSemanticsWitnessMask = + ThreadSemanticsWitnessUseB | ThreadSemanticsWitnessMixed; + +static_assert(ThreadSemanticsElementBytes == 2, + "Thread semantics cases encode F16 elements"); +static_assert(ThreadSemanticsM * ThreadSemanticsStrideBytes <= + ThreadSemanticsMatrixSlotBytes, + "Each thread's matrix must fit its aligned slot"); +static_assert(ThreadSemanticsWitnessOffset % sizeof(uint32_t) == 0 && + ThreadSemanticsArmOffset % sizeof(uint32_t) == 0, + "Witness words must be four byte aligned"); + +static int threadSemanticsRowSeed(int Thread, int Row) { + return ThreadSemanticsM * Thread + Row + 1; +} + +// Matrix[t][m][c] = Z + c, vector A is {1..N} and vector B is all -1, which +// keeps the two arms' results in disjoint sign ranges. +static int threadSemanticsMatrixValue(int Thread, int Row, int Column) { + return threadSemanticsRowSeed(Thread, Row) + Column; +} + +// Closed forms over sum(c+1) = 36, sum(c*(c+1)) = 168 and sum(c) = 28: +// A: sum_c (Z + c) * (c + 1) = 36*Z + 168 +// B: sum_c -(Z + c) = -(8*Z + 28) +static int threadSemanticsExpected(int Thread, int Row, bool UseVectorB) { + static_assert(ThreadSemanticsN == 8, + "Closed-form expectations are derived for N = 8"); + const int Z = threadSemanticsRowSeed(Thread, Row); + return UseVectorB ? -(8 * Z + 28) : 36 * Z + 168; +} + +static void storeThreadSemanticsHalf(std::vector &Bytes, size_t Offset, + int Value) { + const HLSLHalf_t Encoded(static_cast(Value)); + VERIFY_ARE_EQUAL(static_cast(Encoded), static_cast(Value)); + std::memcpy(Bytes.data() + Offset, &Encoded, ThreadSemanticsElementBytes); +} + +static std::vector buildThreadSemanticsMatrixBuffer() { + std::vector Bytes( + ThreadSemanticsThreads * ThreadSemanticsMatrixSlotBytes, 0); + for (int Thread = 0; Thread < ThreadSemanticsThreads; ++Thread) + for (int Row = 0; Row < ThreadSemanticsM; ++Row) + for (int Column = 0; Column < ThreadSemanticsN; ++Column) + storeThreadSemanticsHalf( + Bytes, + Thread * ThreadSemanticsMatrixSlotBytes + + Row * ThreadSemanticsStrideBytes + + Column * ThreadSemanticsElementBytes, + threadSemanticsMatrixValue(Thread, Row, Column)); + return Bytes; +} + +static std::vector buildThreadSemanticsVectorBuffer() { + std::vector Bytes(2 * ThreadSemanticsVectorBytes, 0); + for (int Column = 0; Column < ThreadSemanticsN; ++Column) { + storeThreadSemanticsHalf(Bytes, Column * ThreadSemanticsElementBytes, + Column + 1); + storeThreadSemanticsHalf( + Bytes, + ThreadSemanticsVectorBytes + Column * ThreadSemanticsElementBytes, -1); + } + return Bytes; +} + +enum class ThreadSemanticsOutcome { Verified, Inconclusive, Failed }; + +static ThreadSemanticsOutcome verifyThreadSemanticsOutput(const void *Data, + size_t Size, + bool Divergent, + bool Verbose) { + const size_t RequiredBytes = + ThreadSemanticsThreads * ThreadSemanticsOutputSlotBytes; + if (Size < RequiredBytes) { + hlsl_test::LogErrorFmt( + L"Thread semantics readback is %zu bytes, expected at least %zu", Size, + RequiredBytes); + return ThreadSemanticsOutcome::Failed; + } + + const BYTE *Bytes = static_cast(Data); + bool Success = true; + int MixedWaves = 0; + for (int Thread = 0; Thread < ThreadSemanticsThreads; ++Thread) { + const BYTE *Slot = Bytes + Thread * ThreadSemanticsOutputSlotBytes; + uint32_t Witness = 0; + uint32_t Arm = 0; + std::memcpy(&Witness, Slot + ThreadSemanticsWitnessOffset, sizeof(Witness)); + std::memcpy(&Arm, Slot + ThreadSemanticsArmOffset, sizeof(Arm)); + + if ((Witness & ~ThreadSemanticsWitnessMask) != 0) { + hlsl_test::LogErrorFmt(L"Thread %d witness has unexpected bits: 0x%08x", + Thread, Witness); + Success = false; + continue; + } + if (Arm > 1) { + hlsl_test::LogErrorFmt(L"Thread %d arm marker is not 0 or 1: 0x%08x", + Thread, Arm); + Success = false; + continue; + } + + if ((Witness & ThreadSemanticsWitnessMixed) != 0) + ++MixedWaves; + + // The witness is written before the branch and the arm marker inside it, so + // a disagreement means the arm that ran is not the one the predicate chose. + const bool UseVectorB = (Witness & ThreadSemanticsWitnessUseB) != 0; + if (UseVectorB != (Arm == 1)) { + hlsl_test::LogErrorFmt( + L"Thread %d executed arm %s but its predicate selected vector %s", + Thread, Arm == 1 ? L"B" : L"A", UseVectorB ? L"B" : L"A"); + Success = false; + continue; + } + if (!Divergent && UseVectorB) { + hlsl_test::LogErrorFmt(L"Thread %d selected vector B without a branch", + Thread); + Success = false; + continue; + } + + const HLSLHalf_t *Values = reinterpret_cast(Slot); + for (int Row = 0; Row < ThreadSemanticsM; ++Row) { + const float Actual = static_cast(Values[Row]); + const float Expected = + static_cast(threadSemanticsExpected(Thread, Row, UseVectorB)); + // Every expectation is an integer F16 holds exactly. + if (Actual != Expected) { + hlsl_test::LogErrorFmt(L"Thread %d row %d (vector %s): actual=%f, " + L"expected=%f", + Thread, Row, UseVectorB ? L"B" : L"A", + static_cast(Actual), + static_cast(Expected)); + Success = false; + } else if (Verbose) + hlsl_test::LogCommentFmt(L"Thread %d row %d (vector %s): %f", Thread, + Row, UseVectorB ? L"B" : L"A", + static_cast(Actual)); + } + } + + if (!Success) + return ThreadSemanticsOutcome::Failed; + + // Thread-to-wave distribution is implementation defined, so a wave holding + // only same-parity lanes is legal and leaves the divergent case unproven + // rather than violated. + if (Divergent && MixedWaves == 0) { + hlsl_test::LogCommentFmt( + L"Every thread reported a wave that executed a single arm, so the " + L"thread-scope operations never ran under non-uniform control flow " + L"and this case establishes nothing about it. The per-thread results " + L"were verified before reporting this case as inconclusive"); + return ThreadSemanticsOutcome::Inconclusive; + } + if (Divergent && Verbose) + hlsl_test::LogCommentFmt(L"%d of %d threads ran in a wave that executed " + L"both arms", + MixedWaves, ThreadSemanticsThreads); + + return ThreadSemanticsOutcome::Verified; +} + +static const char ThreadPerLaneMatVecShader[] = R"( + ByteAddressBuffer MatrixInput : register(t0); + ByteAddressBuffer VectorInput : register(t1); + RWByteAddressBuffer Output : register(u2); + + [numthreads(NUMTHREADS, 1, 1)] + void main(uint3 GroupThreadID : SV_GroupThreadID) { + const uint T = GroupThreadID.x; + + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, T * MATRIX_SLOT_BYTES, STRIDE, LAYOUT, + MATRIX_SLOT_BYTES); + + vector InVec; + for (uint I = 0; I < N_DIM; ++I) { + InVec[I] = VectorInput.Load(I * ELEM_SIZE); + } + + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiply(OutVec, Mat, 1, InVec, IN_INTERP); + + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(T * OUTPUT_SLOT_BYTES + I * ELEM_SIZE, OutVec[I]); + } + Output.Store(T * OUTPUT_SLOT_BYTES + WITNESS_OFFSET, 0); + Output.Store(T * OUTPUT_SLOT_BYTES + ARM_OFFSET, 0); + } +)"; + +static const char ThreadDivergentMatVecShader[] = R"( + ByteAddressBuffer MatrixInput : register(t0); + ByteAddressBuffer VectorInput : register(t1); + RWByteAddressBuffer Output : register(u2); + + [numthreads(NUMTHREADS, 1, 1)] + void main(uint3 GroupThreadID : SV_GroupThreadID) { + const uint T = GroupThreadID.x; + vector OutVec; + + // Lane parity rather than thread parity, because the thread-to-wave + // mapping is implementation defined and only lane parity is guaranteed to + // diverge inside a wave. Both votes run with every lane active. + const bool UseB = (WaveGetLaneIndex() & 1) != 0; + const bool Mixed = WaveActiveAnyTrue(UseB) && WaveActiveAnyTrue(!UseB); + Output.Store(T * OUTPUT_SLOT_BYTES + WITNESS_OFFSET, + (UseB ? WITNESS_USE_B : 0) | (Mixed ? WITNESS_MIXED : 0)); + + if (!UseB) { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, T * MATRIX_SLOT_BYTES, STRIDE, LAYOUT, + MATRIX_SLOT_BYTES); + + vector VecA; + for (uint I = 0; I < N_DIM; ++I) { + VecA[I] = VectorInput.Load(I * ELEM_SIZE); + } + __builtin_LinAlg_MatrixVectorMultiply(OutVec, Mat, 1, VecA, IN_INTERP); + Output.Store(T * OUTPUT_SLOT_BYTES + ARM_OFFSET, 0); + } else { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, T * MATRIX_SLOT_BYTES, STRIDE, LAYOUT, + MATRIX_SLOT_BYTES); + + vector VecB; + for (uint I = 0; I < N_DIM; ++I) { + VecB[I] = VectorInput.Load(VEC_B_OFFSET + I * ELEM_SIZE); + } + __builtin_LinAlg_MatrixVectorMultiply(OutVec, Mat, 1, VecB, IN_INTERP); + Output.Store(T * OUTPUT_SLOT_BYTES + ARM_OFFSET, 1); + } + + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(T * OUTPUT_SLOT_BYTES + I * ELEM_SIZE, OutVec[I]); + } + } +)"; + +static MatrixParams makeThreadSemanticsParams() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = ThreadSemanticsM; + Params.N = ThreadSemanticsN; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Thread; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = ThreadSemanticsThreads; + Params.Enable16Bit = true; + return Params; +} + +static void runThreadSemanticsMatVec(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, + const char *Shader, bool Divergent, + bool Verbose) { + VERIFY_ARE_EQUAL(Params.strideBytes(), ThreadSemanticsStrideBytes); + + std::stringstream ExtraDefs; + ExtraDefs << " -DIN_INTERP=" << static_cast(Params.CompType); + ExtraDefs << " -DMATRIX_SLOT_BYTES=" << ThreadSemanticsMatrixSlotBytes; + ExtraDefs << " -DOUTPUT_SLOT_BYTES=" << ThreadSemanticsOutputSlotBytes; + ExtraDefs << " -DVEC_B_OFFSET=" << ThreadSemanticsVectorBytes; + ExtraDefs << " -DWITNESS_OFFSET=" << ThreadSemanticsWitnessOffset; + ExtraDefs << " -DARM_OFFSET=" << ThreadSemanticsArmOffset; + ExtraDefs << " -DWITNESS_USE_B=" << ThreadSemanticsWitnessUseB; + ExtraDefs << " -DWITNESS_MIXED=" << ThreadSemanticsWitnessMixed; + + const std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + compileShader(DxcSupport, Shader, "cs_6_10", Args, Verbose); + + const std::vector MatrixBuffer = buildThreadSemanticsMatrixBuffer(); + const std::vector VectorBuffer = buildThreadSemanticsVectorBuffer(); + const size_t OutputBytes = + ThreadSemanticsThreads * ThreadSemanticsOutputSlotBytes; + + auto Op = createComputeOp(Shader, "cs_6_10", "SRV(t0), SRV(t1), UAV(u2)", + Args.c_str()); + addSRVBuffer(Op.get(), "MatrixInput", MatrixBuffer.size(), "byname"); + addSRVBuffer(Op.get(), "VectorInput", VectorBuffer.size(), "byname"); + addUAVBuffer(Op.get(), "Output", OutputBytes, true, "byname"); + addRootView(Op.get(), 0, "MatrixInput"); + addRootView(Op.get(), 1, "VectorInput"); + addRootView(Op.get(), 2, "Output"); + + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [&](LPCSTR Name, std::vector &Data, st::ShaderOp *) { + if (_stricmp(Name, "Output") == 0) { + cpu_oracle::fillPoison(Data.data(), Data.size()); + return; + } + const std::vector *Source = nullptr; + if (_stricmp(Name, "MatrixInput") == 0) + Source = &MatrixBuffer; + else if (_stricmp(Name, "VectorInput") == 0) + Source = &VectorBuffer; + VERIFY_IS_TRUE(Source != nullptr, + "Unexpected thread semantics resource initializer"); + if (!Source) + return; + VERIFY_ARE_EQUAL(Data.size(), Source->size()); + if (Data.size() == Source->size()) + std::memcpy(Data.data(), Source->data(), Data.size()); + }); + + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); + switch (verifyThreadSemanticsOutput(OutData.data(), OutData.size(), Divergent, + Verbose)) { + case ThreadSemanticsOutcome::Verified: + return; + case ThreadSemanticsOutcome::Inconclusive: + WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); + return; + case ThreadSemanticsOutcome::Failed: + VERIFY_IS_TRUE(false, "Thread semantics verification failed"); + return; + } + VERIFY_IS_TRUE(false, "Unknown thread semantics outcome"); +} + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F16_PerThread() { + const MatrixParams Params = makeThreadSemanticsParams(); + + if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F16, + /*HasBias=*/false, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_4x8_F16_PerThread")) + return; + + runThreadSemanticsMatVec(D3DDevice, DxcSupport, Params, + ThreadPerLaneMatVecShader, /*Divergent=*/false, + VerboseLogging); +} + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F16_Divergent() { + const MatrixParams Params = makeThreadSemanticsParams(); + + if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F16, + /*HasBias=*/false, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_4x8_F16_Divergent")) + return; + + runThreadSemanticsMatVec(D3DDevice, DxcSupport, Params, + ThreadDivergentMatVecShader, /*Divergent=*/true, + VerboseLogging); +} + static const char MatVecMulAddShader[] = R"( #define USE_A 0 #define SCOPE_THREAD 0