diff --git a/clients/gtest/matmul_gtest.yaml b/clients/gtest/matmul_gtest.yaml index 71e88fc94b..ee611e2325 100644 --- a/clients/gtest/matmul_gtest.yaml +++ b/clients/gtest/matmul_gtest.yaml @@ -1247,7 +1247,7 @@ Tests: beta: [ 0.0, 2.0 ] unit_check: 1 algo_method: [2] - gpu_arch: '942' + gpu_arch: '9(42 | 50)' - name: matmul_groupedgemm_zero_n category: pre_checkin @@ -1648,7 +1648,7 @@ Tests: alpha: 1 beta: [ 0.0, 2.0 ] unit_check: 1 - gpu_arch: '942' + gpu_arch: '9(42 | 50)' - name: matmul_gemm_double category: pre_checkin diff --git a/clients/include/allclose.hpp b/clients/include/allclose.hpp index 790414ffec..eef393d770 100644 --- a/clients/include/allclose.hpp +++ b/clients/include/allclose.hpp @@ -48,8 +48,14 @@ /*! \brief compare the allclose error of two matrices hCPU & hGPU */ template -bool allclose(size_t* N, T* a, T* b, double atol, double rtol, bool equal_nan = false) +bool allclose(size_t* N, T* a, T* b, double atol, double rtol, + bool equal_nan, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { + bool success = true; for(size_t i = 0; i < *N; i++) { //Returning ture immediately if 2 elements are identical. @@ -60,14 +66,21 @@ bool allclose(size_t* N, T* a, T* b, double atol, double rtol, bool equal_nan = if(equal_nan && (std::isnan(a[i]) && std::isnan(b[i]))) continue; if(equal_nan && (std::isnan(a[i]) ^ std::isnan(b[i]))) - return false; + success = false; double error = std::abs(a[i] - b[i]); + double relError = (std::abs(a[i]) < 1e-06) ? 0.0 : error / std::abs(a[i]) + 1e-06; double tolerance = atol + std::abs(rtol * b[i]); if(!(error <= tolerance)) - return false; + success = false; + absErrorAvg += error; + relErrorAvg += relError; + absErrorMax = std::max(error, absErrorMax); + relErrorMax = std::max(relError, relErrorMax); } - return true; + absErrorAvg = absErrorAvg / double(*N); + relErrorAvg = relErrorAvg / double(*N); + return success; } template < @@ -85,7 +98,12 @@ bool allclose_check_general(char allclose_type, T* hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg + ) { if(M * N == 0) return 0; @@ -109,7 +127,8 @@ bool allclose_check_general(char allclose_type, { for(auto& rtol : rtols) { - if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false)) + if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg)) { hipblaslt_atol = atol; hipblaslt_rtol = rtol; @@ -141,7 +160,11 @@ bool allclose_check_general(char allclose_type, T* hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -165,7 +188,8 @@ bool allclose_check_general(char allclose_type, { for(auto& rtol : rtols) { - if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false)) + if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg)) { hipblaslt_atol = atol; hipblaslt_rtol = rtol; @@ -197,7 +221,11 @@ bool allclose_check_general(char allclose_type, T* hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -221,7 +249,8 @@ bool allclose_check_general(char allclose_type, { for(auto& rtol : rtols) { - if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false)) + if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg)) { hipblaslt_atol = atol; hipblaslt_rtol = rtol; @@ -254,7 +283,11 @@ bool allclose_check_general(char allclose_type, VEC&& hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -273,7 +306,8 @@ bool allclose_check_general(char allclose_type, } return allclose_check_general( - allclose_type, M, N, lda, hCPU_double, hGPU_double, hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU_double, hGPU_double, hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); } // For int8, we convert the results to int first @@ -285,7 +319,11 @@ bool allclose_check_general(char allclose_type, VEC&& hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -304,7 +342,8 @@ bool allclose_check_general(char allclose_type, } return allclose_check_general( - allclose_type, M, N, lda, hCPU_int, hGPU_int, hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU_int, hGPU_int, hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); } /* ============== allclose check for strided_batched case ============= */ @@ -318,7 +357,11 @@ bool allclose_check_general(char allclose_type, T* hGPU, int64_t batch_count, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -327,7 +370,8 @@ bool allclose_check_general(char allclose_type, { auto index = i * stride_a; bool close = allclose_check_general( - allclose_type, M, N, lda, hCPU + index, hGPU + index, hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU + index, hGPU + index, hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); if(!close) return false; } @@ -345,7 +389,11 @@ bool allclose_check_general(char allclose_type, T* hGPU[], int64_t batch_count, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -354,7 +402,8 @@ bool allclose_check_general(char allclose_type, { auto index = i; bool close = allclose_check_general( - allclose_type, M, N, lda, hCPU[index], hGPU[index], hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU[index], hGPU[index], hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); if(!close) return false; } @@ -372,7 +421,11 @@ bool allclose_check_general(char allclose_type, int64_t batch_count, double& hipblaslt_atol, double& hipblaslt_rtol, - hipDataType type) + hipDataType type, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { switch(type) { @@ -386,7 +439,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_64F: return allclose_check_general(allclose_type, M, @@ -397,7 +454,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_16F: return allclose_check_general(allclose_type, M, @@ -408,7 +469,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_16BF: return allclose_check_general(allclose_type, M, @@ -419,7 +484,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8F_E4M3_FNUZ: return allclose_check_general(allclose_type, M, @@ -430,7 +499,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8F_E5M2_FNUZ: return allclose_check_general(allclose_type, M, @@ -441,7 +514,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); #ifdef ROCM_USE_FLOAT8 case HIP_R_8F_E4M3: return allclose_check_general(allclose_type, @@ -453,7 +530,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8F_E5M2: return allclose_check_general(allclose_type, M, @@ -464,7 +545,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); #endif case HIP_R_32I: return allclose_check_general(allclose_type, @@ -476,7 +561,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8I: return allclose_check_general(allclose_type, M, @@ -487,7 +576,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); default: hipblaslt_cerr << "Error type in allclose_check_general" << std::endl; return false; diff --git a/clients/include/argument_model.hpp b/clients/include/argument_model.hpp index 1cc64860a7..f59611bed1 100644 --- a/clients/include/argument_model.hpp +++ b/clients/include/argument_model.hpp @@ -72,7 +72,11 @@ class ArgumentModel double cpu_us, double norm, double atol, - double rtol) + double rtol, + double absErrorMax, + double absErrorAvg, + double relErrorMax, + double relErrorAvg) { // requires enablement for frequency logging ArgumentModel_log_performance(name_line, val_line); @@ -154,6 +158,11 @@ class ArgumentModel else val_line << "," << rtol; } + if (absErrorMax != ArgumentLogging::NA_value) + { + name_line << ",absErrorMax,absErrorAvg,relErrorMax,relErrorAvg"; + val_line << "," << absErrorMax << "," << absErrorAvg << "," << relErrorMax << "," << relErrorAvg; + } } } } @@ -176,7 +185,11 @@ class ArgumentModel double cpu_us = ArgumentLogging::NA_value, double norm = ArgumentLogging::NA_value, double atol = ArgumentLogging::NA_value, - double rtol = ArgumentLogging::NA_value) + double rtol = ArgumentLogging::NA_value, + double absErrorMax = ArgumentLogging::NA_value, + double absErrorAvg = ArgumentLogging::NA_value, + double relErrorMax = ArgumentLogging::NA_value, + double relErrorAvg = ArgumentLogging::NA_value) { hipblaslt_internal_ostream name_list; hipblaslt_internal_ostream value_list; @@ -261,7 +274,11 @@ class ArgumentModel cpu_us, norm, atol, - rtol); + rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); if(archName != "") { diff --git a/clients/include/testing_matmul.hpp b/clients/include/testing_matmul.hpp index 2eeeeb6995..388420f87c 100644 --- a/clients/include/testing_matmul.hpp +++ b/clients/include/testing_matmul.hpp @@ -839,7 +839,11 @@ void check(hipStream_t stream, hipDataType To, hipDataType Tbias, hipDataType Taux, - hipDataType Tc) + hipDataType Tc, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { // fetch GPU CHECK_HIP_ERROR(hipStreamSynchronize(stream)); @@ -1048,7 +1052,11 @@ void check(hipStream_t stream, num_batches[gemmIdx], hipblaslt_atol, hipblaslt_rtol, - To); + To, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); //TODO: confirm if allclose_check_assert is neccessary } } @@ -3343,6 +3351,11 @@ void testing_matmul_with_bias(const Arguments& arg, } } + double absErrorMax = 0; + double absErrorAvg = 0; + double relErrorMax = 0; + double relErrorAvg = 0; + if(!arg.timing) { for(size_t sol = 0; sol < heuristicResult.size(); sol++) @@ -3463,7 +3476,11 @@ void testing_matmul_with_bias(const Arguments& arg, To, Tbias, Taux, - Talpha); + Talpha, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } } } @@ -3866,7 +3883,11 @@ void testing_matmul_with_bias(const Arguments& arg, To, Tbias, Taux, - Talpha); + Talpha, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } #define argument_param \ @@ -3935,7 +3956,11 @@ void testing_matmul_with_bias(const Arguments& arg, cpu_time_used, hipblaslt_error, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } if(best_gpu_time > gpu_time_used) { @@ -3993,7 +4018,11 @@ void testing_matmul_with_bias(const Arguments& arg, cpu_time_used, best_norm, best_atol, - best_rtol); + best_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } } diff --git a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml index 0a44d8b8a1..5bca355119 100644 --- a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml +++ b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml @@ -89,10 +89,11 @@ AssertSummationElementMultiple: 1 AssignedDerivedParameters: true AssignedProblemIndependentDerivedParameters: true - BaseName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x1J7V_jobmP44OFztrsbXl-d3dSIK0CEdTi8QZcUodlVY= + BaseName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x1S2F3kkilrgK40SskFsdYEE0gPv9CMGaBoFPKzPf7I1Y= BufferLoad: true BufferStore: true CUCount: null + CUOccupancy: -1 ClusterLocalRead: 0 CodeObjectVersion: '4' ConvertAfterDS: false @@ -130,7 +131,7 @@ SupportUserGSU: true, UseUniversalArgs: true} Kernel: true KernelLanguage: Assembly - KernelNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_K1_MIWT1_1 + KernelNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA4_GRVWB1_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA1024_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA1_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG64_4_1 LDSTrInst: false LSCA: 64 LSCB: 16 @@ -188,6 +189,7 @@ MacroTileA: 64 MacroTileB: 16 MagicDivAlg: 2 + MathClocksUnrolledLoop: 0 MatrixInstB: 1 MatrixInstBM: 1 MatrixInstBN: 1 @@ -226,7 +228,7 @@ PackedC1IndicesX: [1] PrefetchGlobalRead: 1 PrefetchLocalRead: 0 - PreloadKernArgs: 0 + PreloadKernArgs: true ProblemType: Activation: false ActivationComputeDataType: 0 @@ -308,13 +310,14 @@ ScheduleIterAlg: 1 ScheduleLocalWrite: 1 SolutionIndex: 0 - SolutionNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_GSU1_GSUC0_GSUWGMRR0_K1_MIWT1_1_SU32_SUM0_SUS256_WGM8_WGMXCC1_WGMXCCGn1 + SolutionNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA4_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA1024_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA1_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG64_4_1_WGM8_WGMXCC1_WGMXCCGn1 SourceSwap: 1 StaggerU: 32 StaggerUMapping: 0 StaggerUStride: 256 StorePriorityOpt: false StoreRemapVectorWidth: 0 + StoreSwapAddr: false StoreSyncOpt: 0 StoreVectorWidth: 1 StreamK: 0 @@ -339,7 +342,7 @@ UnrollMajorLDSMetadata: true Use64bShadowLimit: 1 UseDotInstruction: false - UseF32XEmulation: false + UseF32XEmulation: true UseInstOffsetForGRO: 0 UseSgprForGRO: -1 Valid: true @@ -375,9 +378,7 @@ tailLoopOptB: false - [2, 3, 0, 1] - - - [128, 128, 1, 128, 128, 128, 128, 128] - - [0, 0.19] - - - [2048, 2048, 1, 256, 2048, 2048, 2048, 2048] - - [0, 15.01] + - [0, 0.35] - null - null - DeviceEfficiency diff --git a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml index 139de69e69..dfa079ec90 100644 --- a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml +++ b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml @@ -89,36 +89,38 @@ AssertSummationElementMultiple: 1 AssignedDerivedParameters: true AssignedProblemIndependentDerivedParameters: true - BaseName: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT64x16x16_MI16x1J7V_jobmP44OFztrsbXl-d3dSIK0CEdTi8QZcUodlVY= + BaseName: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT256x256x32_MI16GZB8tvR2VQN0ZSch4g4iOm8t5mZh5aOHgP_1vzk34Bo= BufferLoad: true BufferStore: true CUCount: null + CUOccupancy: -1 ClusterLocalRead: 0 CodeObjectVersion: '4' ConvertAfterDS: false CustomKernelName: '' DebugStreamK: 0 - DepthU: 16 - DirectToLds: false + DepthU: 32 + DirectToLds: 0 DirectToLdsA: false DirectToLdsB: false DirectToVgprA: false DirectToVgprB: false DirectToVgprSparseMetadata: false EdgeType: ShiftPtr - EnableF32XEmulationLds: false + EnableF32XEmulationLds: true EnableF32XdlMathOp: true EnableMatrixInstruction: true - ExpandPointerSwap: true + ExpandPointerSwap: 0 + ExpertSchedulingMode: 0 ForceDisableShadowInit: false GlobalReadPerMfma: 1 GlobalReadVectorWidthA: 4 - GlobalReadVectorWidthB: 1 + GlobalReadVectorWidthB: 4 GlobalSplitU: 1 GlobalSplitUAlgorithm: MultipleBuffer GlobalSplitUCoalesced: false GlobalSplitUWorkGroupMappingRoundRobin: false - GlobalWriteVectorWidth: 1 + GlobalWriteVectorWidth: 4 GroupLoadStore: false GuaranteeNoPartialA: true GuaranteeNoPartialB: true @@ -130,34 +132,34 @@ SupportUserGSU: true, UseUniversalArgs: true} Kernel: true KernelLanguage: Assembly - KernelNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_K1_MIWT1_1 + KernelNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT256x256x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB512_LBSPPM0_LPA8_LPB8_LPM0_LRVW4_LWPMn1_MIAV0_MIWT8_8_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW4_SK0_SKXCCM0_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA4_VWB4_WSGRA0_WSGRB0_WS64_WG32_8_1 LDSTrInst: false - LSCA: 16 - LSCB: 16 - LSPA: 64 - LSPB: 16 - LVCA: 4 - LVCB: 16 - LVPA: 16 - LVPB: 16 - LdsBlockSizePerPadA: 128 - LdsBlockSizePerPadB: 128 + LSCA: 32 + LSCB: 32 + LSPA: 32 + LSPB: 32 + LVCA: 8 + LVCB: 8 + LVPA: 8 + LVPB: 8 + LdsBlockSizePerPadA: 512 + LdsBlockSizePerPadB: 512 LdsBlockSizePerPadMetadata: 0 - LdsBytesNoAmax: 14592 + LdsBytesNoAmax: 139264 LdsInitCVgprs: false - LdsNumBytes: 14592 - LdsNumElementsAlignedA: 5120 - LdsNumElementsAlignedB: 1280 + LdsNumBytes: 139264 + LdsNumElementsAlignedA: 34816 + LdsNumElementsAlignedB: 34816 LdsNumElementsAlignedMetadata: 0 LdsOffsetA: 0 - LdsOffsetA_Blk: 8192 - LdsOffsetB: 5120 - LdsOffsetB_Blk: 13312 + LdsOffsetA_Blk: 69632 + LdsOffsetB: 34816 + LdsOffsetB_Blk: 104448 LdsOffsetBias: 0 LdsOffsetBiasGSU: 0 LdsOffsetBiasNonGSU: 0 - LdsOffsetMetadata: 5120 - LdsOffsetMetadata_Blk: 13312 + LdsOffsetMetadata: 34816 + LdsOffsetMetadata_Blk: 104448 LdsPadA: 8 LdsPadB: 8 LdsPadMetadata: 0 @@ -168,33 +170,34 @@ LocalWriteUseSgprA: false LocalWriteUseSgprB: false LoopIters: 1 - LoopUnroll: 16 + LoopUnroll: 32 MFMA_BF16_1K: false MIArchVgpr: false - MIBlock: [16, 16, 16, 1, 1, 1] - MIInputPerThread: 4 - MIInputPerThreadA: 4 - MIInputPerThreadB: 4 - MIInputPerThreadMetadata: 4 + MIBlock: [16, 16, 32, 1, 1, 1] + MIInputPerThread: 8 + MIInputPerThreadA: 8 + MIInputPerThreadB: 8 + MIInputPerThreadMetadata: 8 MIOutputVectorWidth: 4 MIRegPerOut: 1 - MIWaveGroup: [4, 1] - MIWaveTile: [1, 1] - MIWaveTileA: 1 - MIWaveTileB: 1 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 8] + MIWaveTileA: 8 + MIWaveTileB: 8 MIWaveTileMetadata: 0 - MacroTile0: 64 - MacroTile1: 16 - MacroTileA: 64 - MacroTileB: 16 + MacroTile0: 256 + MacroTile1: 256 + MacroTileA: 256 + MacroTileB: 256 MagicDivAlg: 2 + MathClocksUnrolledLoop: 0 MatrixInstB: 1 MatrixInstBM: 1 MatrixInstBN: 1 - MatrixInstK: 16 + MatrixInstK: 32 MatrixInstM: 16 MatrixInstN: 16 - MatrixInstruction: [16, 16, 16, 1] + MatrixInstruction: [16, 16, 32, 1] MaxOccupancy: 40 MbskPrefetchOpt: 0 NoLdsWriteCode: false @@ -209,14 +212,14 @@ NonTemporalMetadata: 0 NonTemporalWS: 0 NumElementsPerBatchStore: 0 - NumElementsPerThread: 4 - NumGlobalWriteVectorsPerThread: 4 - NumLoadsA: 1 - NumLoadsB: 1 + NumElementsPerThread: 256 + NumGlobalWriteVectorsPerThread: 64 + NumLoadsA: 8 + NumLoadsB: 8 NumLoadsCoalescedA: 1 NumLoadsCoalescedB: 1 - NumLoadsPerpendicularA: 1 - NumLoadsPerpendicularB: 1 + NumLoadsPerpendicularA: 8 + NumLoadsPerpendicularB: 8 NumThreads: 256 NumWaveSplitK: 1 OptNoLoadLoop: 1 @@ -224,9 +227,9 @@ PackedC0IndicesX: [0] PackedC1IdxChars: [J] PackedC1IndicesX: [1] - PrefetchGlobalRead: 1 + PrefetchGlobalRead: 2 PrefetchLocalRead: 0 - PreloadKernArgs: 0 + PreloadKernArgs: true ProblemType: Activation: false ActivationComputeDataType: 0 @@ -305,31 +308,32 @@ UseScaleAlphaVec: 0 UseScaleCD: false ScheduleGlobalRead: 1 - ScheduleIterAlg: 1 + ScheduleIterAlg: 3 ScheduleLocalWrite: 1 SolutionIndex: 0 - SolutionNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_GSU1_GSUC0_GSUWGMRR0_K1_MIWT1_1_SU32_SUM0_SUS256_WGM8_WGMXCC1_WGMXCCGn1 + SolutionNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT256x256x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB512_LBSPPM0_LPA8_LPB8_LPM0_LRVW4_LWPMn1_MIAV0_MIWT8_8_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SU0_SUM0_SUS128_SPO0_SRVW0_SSO0_SVW4_SK0_SKXCCM0_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA4_VWB4_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1 SourceSwap: 1 - StaggerU: 32 + StaggerU: 0 StaggerUMapping: 0 - StaggerUStride: 256 + StaggerUStride: 128 StorePriorityOpt: false StoreRemapVectorWidth: 0 + StoreSwapAddr: true StoreSyncOpt: 0 - StoreVectorWidth: 1 + StoreVectorWidth: 4 StreamK: 0 StreamKAtomic: 0 StreamKXCCMapping: 0 - SubGroup0: 16 - SubGroup1: 16 - SubGroupA: 16 - SubGroupB: 16 + SubGroup0: 8 + SubGroup1: 32 + SubGroupA: 8 + SubGroupB: 32 SuppressNoLoadLoop: false ThreadTile: [1, 1] - ThreadTile0: 4 - ThreadTile1: 1 - ThreadTileA: 4 - ThreadTileB: 1 + ThreadTile0: 32 + ThreadTile1: 8 + ThreadTileA: 32 + ThreadTileB: 8 TransposeLDS: 1 TransposeLDSMetadata: true ULSGRODoubleG2L: 0 @@ -339,34 +343,34 @@ UnrollMajorLDSMetadata: true Use64bShadowLimit: 1 UseDotInstruction: false - UseF32XEmulation: false + UseF32XEmulation: true UseInstOffsetForGRO: 0 UseSgprForGRO: -1 Valid: true VectorStore: -1 - VectorWidthA: 1 - VectorWidthB: 1 + VectorWidthA: 4 + VectorWidthB: 4 WaveSeparateGlobalReadA: 0 WaveSeparateGlobalReadB: 0 WaveSeparateGlobalReadMetadata: 0 WaveSplitK: false WavefrontSize: 64 - WorkGroup: [64, 4, 1] + WorkGroup: [32, 8, 1] WorkGroupMapping: 8 WorkGroupMappingXCC: 1 WorkGroupMappingXCCGroup: -1 WorkGroupReduction: false WorkspaceCheck: [4, 0, 1] - _DepthU: 16 - _DepthUA: 16 - _DepthUB: 16 - _DepthUMetadata: 16 + _DepthU: 32 + _DepthUA: 32 + _DepthUB: 32 + _DepthUMetadata: 32 _GlobalAccumulation: MultipleBuffer _UseSgprForGRO: 1 _VectorStore: 1 _WorkspaceSizePerElemBias: 0 _WorkspaceSizePerElemC: 4 - _staggerStrideShift: 2 + _staggerStrideShift: 0 enableLDSTrA: false enableLDSTrB: false reorderGRInstForDTVA: false @@ -374,10 +378,8 @@ tailLoopOptA: false tailLoopOptB: false - [2, 3, 0, 1] -- - - [128, 128, 1, 128, 128, 128, 128, 128] - - [0, 0.18] - - - [2048, 2048, 1, 256, 2048, 2048, 256, 256] - - [0, 15.14] +- - - [4096, 4096, 1, 16384, 4096, 4096, 16384, 16384] + - [0, 153.51] - null - null - DeviceEfficiency diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml new file mode 100644 index 0000000000..5ba002b05f --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml @@ -0,0 +1,105 @@ +--- +CodeObjectFiles: [library/TensileLibrary_gfx950.co] +ConstantParams: + 1LDSBuffer: -1 + ActivationAlt: false + ActivationFuncCall: true + ActivationFused: true + AssertAIGreaterThanEqual: -1 + AssertAILessThanEqual: -1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 1 + BufferLoad: true + BufferStore: true + ClusterLocalRead: 1 + ConvertAfterDS: false + CustomKernelName: '' + DebugStreamK: 0 + DepthU: 32 + DirectToLds: false + DirectToVgprA: false + DirectToVgprB: false + DirectToVgprSparseMetadata: false + ExpandPointerSwap: true + ForceDisableShadowInit: false + GlobalReadPerMfma: 1 + GlobalReadVectorWidthA: 8 + GlobalReadVectorWidthB: -1 + GlobalSplitU: 1 + GlobalSplitUAlgorithm: MultipleBuffer + GlobalSplitUCoalesced: false + GlobalSplitUWorkGroupMappingRoundRobin: false + GroupLoadStore: false + InnerUnroll: 1 + InterleaveAlpha: 0 + KernelLanguage: Assembly + LDSTrInst: false + LdsBlockSizePerPadA: 512 + LdsBlockSizePerPadB: -1 + LdsBlockSizePerPadMetadata: 0 + LdsPadA: -1 + LdsPadB: -1 + LdsPadMetadata: 0 + LocalReadVectorWidth: 4 + LocalWritePerMfma: -1 + MIArchVgpr: false + MagicDivAlg: 2 + MatrixInstruction: [16, 16, 32, 1, 1, 1, 1, 1, 1] + MaxOccupancy: 40 + MbskPrefetchOpt: 0 + NoReject: false + NonTemporal: -1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NonTemporalD: 0 + NonTemporalE: 0 + NonTemporalMetadata: 0 + NonTemporalWS: 0 + NumElementsPerBatchStore: 0 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + OptNoLoadLoop: 1 + PrefetchGlobalRead: 1 + PrefetchLocalRead: 1 + PreloadKernArgs: true + ScheduleGlobalRead: 1 + ScheduleIterAlg: 3 + ScheduleLocalWrite: 1 + SourceSwap: 1 + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + StorePriorityOpt: false + StoreRemapVectorWidth: 0 + StoreSyncOpt: 0 + StoreVectorWidth: -1 + StreamK: 0 + StreamKAtomic: 0 + StreamKXCCMapping: 0 + SuppressNoLoadLoop: false + ThreadTile: [4, 4] + TransposeLDS: -1 + UnrollLoopSwapGlobalReadOrder: 0 + Use64bShadowLimit: 1 + UseInstOffsetForGRO: 0 + UseSgprForGRO: -1 + VectorStore: -1 + VectorWidthA: -1 + VectorWidthB: -1 + WaveSeparateGlobalReadA: 0 + WaveSeparateGlobalReadB: 0 + WaveSeparateGlobalReadMetadata: 0 + WaveSplitK: false + WavefrontSize: 64 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingXCC: 1 + WorkGroupMappingXCCGroup: -1 + WorkGroupReduction: false +CustomKernelWildcard: false +CustomKernels: [] +ForkParams: {} +ParamGroups: [] +... diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini new file mode 100644 index 0000000000..9498e3d2aa --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini @@ -0,0 +1,84 @@ +library-file=/home/cbrownle/git/tests/build_small/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary.yaml +code-object=/home/cbrownle/git/tests/build_small/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary_gfx950.co +results-file=/home/cbrownle/git/tests/build_small/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/../Data/00_Final.csv +performance-metric=DeviceEfficiency +problem-identifier=Contraction_l_Ailk_Bjlk_Cijk_Dijk +compute-input-type=Float +a-type=Float +b-type=Float +c-type=Float +d-type=Float +alpha-type=Float +beta-type=Float +f32-xdl-math-op=XFloat32 +activation-compute-type=Float +use-gradient=False +use-bias=0 +bias-source=3 +use-e=False +output-amaxD=False +use-scaleAB= +use-scaleCD=False +use-scaleAlphaVec=0 +swizzle-tensor-a=False +swizzle-tensor-b=False +icache-flush-args=False +sparse=0 +high-precision-accumulate=False +strided-batched=True +grouped-gemm=False +problem-size=16,16,1,32 +a-strides=-1,16,-1 +b-strides=-1,16,-1 +c-strides=-1,16,-1 +d-strides=-1,16,-1 +activation-type=None +activation-no-guard=False +activation-additional-args=2.0,2.0 +device-idx=0 +init-seed=0 +init-a=SerialIdx +init-b=Identity +init-c=Random +init-d=Zero +init-e=Zero +init-alpha=One +init-beta=Zero +init-bias=Random +init-scaleA=Two +init-scaleB=Two +init-scaleC=Two +init-scaleD=Two +init-scaleAlphaVec=Random +c-equal-d=False +prune-mode=PruneRandom +bounds-check=GuardPageFront +print-valids=False +print-max=4 +num-benchmarks=1 +num-elements-to-validate=-1 +num-enqueues-per-sync=10 +max-enqueues-per-sync=-1 +num-syncs-per-benchmark=1 +skip-slow-solution-ratio=0.0 +use-gpu-timer=True +hardware-monitor=False +num-warmups=5 +min-flops-per-sync=1 +sleep-percent=300 +perf-l2-read-hits=0.0 +perf-l2-write-hits=0.15 +perf-l2-read-bw-mul=2 +perf-read-efficiency=0.85 +csv-export-extra-cols=False +csv-merge-same-problems=False +log-level=Debug +max-workspace-size=13421772800 +PrintWinnersOnly=False +granularity-threshold=0.0 +pristine-on-gpu=True +library-update-file= +library-update-comment=False +use-user-args=False +rotating-buffer-size=0 +rotating-buffer-mode=0 diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h new file mode 100644 index 0000000000..d0e5553e90 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h @@ -0,0 +1,88 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef KERNEL_HEADER +#define KERNEL_HEADER + +#if Tensile_RUNTIME_LANGUAGE_OCL +#include +#else + +#include "TensileTypes.h" +#include + +#include "ReductionTemplate.h" +#include "memory_gfx.h" + + +__device__ inline int GenDot4(int a, int b, int c) +{ +#if(__hcc_workweek__ >= 19092) || __HIP_CLANG_ONLY__ + union + { + int32_t i; + char4 z; + } va, vb; +#else + typedef struct + { + int c0 : 8, c1 : 8, c2 : 8, c3 : 8; + } C4I8; + union + { + int32_t i; + C4I8 z; + } va, vb; +#endif + va.i = a; + vb.i = b; + +#if(__hcc_workweek__ >= 19092) || __HIP_CLANG_ONLY__ + return amd_mixed_dot(va.z, vb.z, c, true); +} +#else + return c + (vb.z.c3 * va.z.c3 + vb.z.c2 * va.z.c2 + vb.z.c1 * va.z.c1 + vb.z.c0 * va.z.c0); +} +#endif + +#endif // HIP + +typedef _Float16 tensile_half2 __attribute__((ext_vector_type(2))); +typedef _Float16 tensile_half; + +extern "C" __device__ tensile_half2 llvm_fma_v2f16(tensile_half2, + tensile_half2, + tensile_half2) __asm("llvm.fma.v2f16"); + +__device__ inline tensile_half2 + tensile_fmadd_half2(tensile_half2 multiplier, tensile_half2 multiplicand, tensile_half2 addend) +{ + tensile_half2 result; + result = llvm_fma_v2f16(multiplier, multiplicand, addend); + return result; +}; + +#endif diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp new file mode 100644 index 0000000000..806a8539cb --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp @@ -0,0 +1,5078 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#include "Cijk_SS_PostGSU16_VW1.h" + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU16_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + } break; + case 14: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + } break; + case 13: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + } break; + case 12: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + } break; + case 11: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + } break; + case 10: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + } break; + case 9: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + } break; + case 8: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + } break; + case 7: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + } break; + case 6: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + } break; + case 5: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + } break; + case 4: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + } break; + case 3: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + } break; + case 2: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + } break; + case 1: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + } break; + default: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + } break; + } +#if defined(__gfx950__) +#endif + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW1( + argument_Cijk_SS_PostGSU16_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + } break; + case 14: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + } break; + case 13: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + } break; + case 12: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + } break; + case 11: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + } break; + case 10: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + } break; + case 9: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + } break; + case 8: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + } break; + case 7: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + } break; + case 6: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + } break; + case 5: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + } break; + case 4: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + } break; + case 3: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + } break; + case 2: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + } break; + case 1: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + } break; + default: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + } break; + } +#if defined(__gfx950__) +#endif + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU8_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + accum[0] += temp[4]; + accum[0] += temp[5]; + accum[0] += temp[6]; + accum[0] += temp[7]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW1( + argument_Cijk_SS_PostGSU8_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + accum[0] += temp[4]; + accum[0] += temp[5]; + accum[0] += temp[6]; + accum[0] += temp[7]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU4_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW1( + argument_Cijk_SS_PostGSU4_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU2_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW1( + argument_Cijk_SS_PostGSU2_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h new file mode 100644 index 0000000000..0e47c7fe6c --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#pragma once +#include +#include + +#include "KernelHeader.h" + + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU16_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU16_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW1( + argument_Cijk_SS_PostGSU16_VW1 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU8_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU8_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW1( + argument_Cijk_SS_PostGSU8_VW1 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU4_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU4_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW1( + argument_Cijk_SS_PostGSU4_VW1 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU2_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU2_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW1( + argument_Cijk_SS_PostGSU2_VW1 arg) +; diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp new file mode 100644 index 0000000000..283bcc9bf6 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp @@ -0,0 +1,9840 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#include "Cijk_SS_PostGSU16_VW2.h" + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU16_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW2( + argument_Cijk_SS_PostGSU16_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU8_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW2( + argument_Cijk_SS_PostGSU8_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU4_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW2( + argument_Cijk_SS_PostGSU4_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU2_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW2( + argument_Cijk_SS_PostGSU2_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h new file mode 100644 index 0000000000..cdc74bec85 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#pragma once +#include +#include + +#include "KernelHeader.h" + + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU16_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU16_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW2( + argument_Cijk_SS_PostGSU16_VW2 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU8_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU8_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW2( + argument_Cijk_SS_PostGSU8_VW2 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU4_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU4_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW2( + argument_Cijk_SS_PostGSU4_VW2 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU2_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU2_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW2( + argument_Cijk_SS_PostGSU2_VW2 arg) +; diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp new file mode 100644 index 0000000000..42acdd8d9d --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp @@ -0,0 +1,12302 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#include "Cijk_SS_PostGSU16_VW4.h" + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU16_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 accumVec2(accum[2], accum[3]); + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; + accum[2] = accumVec2.x; + accum[3] = accumVec2.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW4( + argument_Cijk_SS_PostGSU16_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 accumVec2(accum[2], accum[3]); + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; + accum[2] = accumVec2.x; + accum[3] = accumVec2.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU8_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[2] += temp[4].z; + accum[3] += temp[4].w; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[2] += temp[5].z; + accum[3] += temp[5].w; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[2] += temp[6].z; + accum[3] += temp[6].w; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + accum[2] += temp[7].z; + accum[3] += temp[7].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW4( + argument_Cijk_SS_PostGSU8_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[2] += temp[4].z; + accum[3] += temp[4].w; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[2] += temp[5].z; + accum[3] += temp[5].w; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[2] += temp[6].z; + accum[3] += temp[6].w; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + accum[2] += temp[7].z; + accum[3] += temp[7].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU4_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW4( + argument_Cijk_SS_PostGSU4_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU2_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW4( + argument_Cijk_SS_PostGSU2_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h new file mode 100644 index 0000000000..d74cc47274 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#pragma once +#include +#include + +#include "KernelHeader.h" + + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU16_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU16_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW4( + argument_Cijk_SS_PostGSU16_VW4 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU8_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU8_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW4( + argument_Cijk_SS_PostGSU8_VW4 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU4_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU4_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW4( + argument_Cijk_SS_PostGSU4_VW4 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU2_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU2_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW4( + argument_Cijk_SS_PostGSU2_VW4 arg) +; diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h new file mode 100644 index 0000000000..ee61de51e9 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h @@ -0,0 +1,151 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once +#include "memory_gfx.h" +#include + +template +struct static_for +{ + template + __host__ __device__ constexpr void operator()(F const& f) const + { + if constexpr(Begin < End) + { + f(Begin); + static_for()(f); + } + } +}; + +constexpr size_t max(size_t a, size_t b) +{ + return (a > b) ? a : b; +} + +template +__device__ inline void + reductionKernel_ijk(DataTypeCompute* in, DataTypeOut* out, int m, int n, int strideJ) +{ + __shared__ DataTypeCompute block[MT0 * MT1 * VW]; + + int mod = hipThreadIdx_x % MT0; + int row = hipThreadIdx_x / MT0; + + int soffset = hipBlockIdx_x * MT0 * VW; + int voffset = mod * VW; + int idx = voffset + soffset; + + int soffsetBytes = soffset * sizeof(DataTypeCompute); + int soffsetBytes_bias = soffset * sizeof(DataTypeOut); + int num_records = strideJ * n * sizeof(DataTypeCompute); + int num_records_bias = m * sizeof(DataTypeOut); + + constexpr size_t sumLength = max((size_t)1, VW - 1); + DataTypeCompute sum[sumLength] = {0}; + if(idx + (VW - 1) < m) + { + for(int i = 0; i < n; i += MT1) + { + int currRow = row + i; + int rowStride = currRow * strideJ + voffset; + DataTypeCompute tmp[VW]; + static_for<0, VW>()([&](int vw) { + buffer_load( + tmp[vw], + reinterpret_cast(const_cast(in)), + (uint32_t)((rowStride + vw) * sizeof(DataTypeCompute)), + (uint32_t)(soffsetBytes), + num_records); + }); + static_for<0, VW>()([&](int vw) { sum[vw] += tmp[vw]; }); + } + static_for<0, VW>()([&](int vw) { block[(mod * MT1 + row) * VW + vw] = sum[vw]; }); + __syncthreads(); + if(hipThreadIdx_x < MT0) + { + for(int i = 1; i < MT1; i += 1) + { + static_for<0, VW>()( + [&](int vw) { sum[vw] += block[(i + hipThreadIdx_x * MT1) * VW + vw]; }); + } + // This is a 1D bias + static_for<0, VW>()([&](int vw) { + buffer_store( + (DataTypeOut)sum[vw], + reinterpret_cast(const_cast(out)), + (uint32_t)(voffset + vw) * sizeof(DataTypeOut), + (uint32_t)(soffsetBytes_bias), + num_records_bias); + }); + } + } + else if(idx < m) + { + for(int i = 0; i < n; i += MT1) + { + int currRow = row + i; + int rowStride = currRow * strideJ + voffset; + constexpr size_t tmpLength = max((size_t)1, VW - 1); + DataTypeCompute tmp[tmpLength]; + static_for<0, VW - 1>()([&](int vw) { + buffer_load( + tmp[vw], + reinterpret_cast(const_cast(in)), + (uint32_t)((rowStride + vw) * sizeof(DataTypeCompute)), + (uint32_t)(soffsetBytes), + num_records); + }); + static_for<0, VW - 1>()([&](int vw) { sum[vw] += tmp[vw]; }); + } + static_for<0, VW - 1>()([&](int vw) { block[(mod * MT1 + row) * VW + vw] = sum[vw]; }); + __syncthreads(); + if(hipThreadIdx_x < MT0) + { + + for(int i = 1; i < MT1; i += 1) + { + static_for<0, VW - 1>()( + [&](int vw) { sum[vw] += block[(i + hipThreadIdx_x * MT1) * VW + vw]; }); + } + // This is a 1D bias + static_for<0, VW - 1>()([&](int vw) { + buffer_store( + (DataTypeOut)sum[vw], + reinterpret_cast(const_cast(out)), + (uint32_t)(voffset + vw) * sizeof(DataTypeOut), + (uint32_t)(soffsetBytes_bias), + num_records_bias); + }); + } + } + else + { + static_for<0, VW>()([&](int vw) { block[(mod * MT1 + row) * VW + vw] = 0; }); + __syncthreads(); + } +} diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h new file mode 100644 index 0000000000..7be8412de7 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h @@ -0,0 +1,763 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef TENSILETYPES_H +#define TENSILETYPES_H +#include "tensile_bfloat16.h" + +#if (HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR == 2 && HIP_VERSION_PATCH > 42130) \ + || (HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR >= 3)//tmp before gfx94 use hip f8 header + +// Using hip header for both NANOO and OCP data types +#if defined(__HIPCC__) +#include +#define TENSILELITE_FP8_TYPE_FNUZ HIP_FP8_TYPE_FNUZ +#define TENSILELITE_FP8_TYPE_OCP HIP_FP8_TYPE_OCP +#endif + +#include "tensile_float8_bfloat8.h" +#else + +#if !defined(HIP_FP8_TYPE_FNUZ) +#define TENSILELITE_FP8_TYPE_FNUZ 1 +#endif + +#if !defined(HIP_FP8_TYPE_OCP) +#define TENSILELITE_FP8_TYPE_OCP 0 +#endif + +#include "tensile_float8_bfloat8_bc.h" +#endif + +#include +#include +#include +#include +#include + +#include + +// OpenCL +#if Tensile_RUNTIME_LANGUAGE_OCL +#include "CL/cl.h" +#define TensileStatus cl_int +#define tensileStatusSuccess CL_SUCCESS +#define tensileStatusFailure -1 +#define tensileStatusAssertFailure -2 +#define TensileComplexFloat cl_float2 +#define TensileComplexDouble cl_double2 +#define TensileHalf cl_half + +// HIP +#else +#include +#define TensileStatus hipError_t +#define tensileStatusSuccess hipSuccess +// FIXME - steal hip error encodings since rocBLAS expects hip error codes to be +// returned... +#define tensileStatusFailure hipErrorUnknown +#define tensileStatusAssertFailure hipErrorRuntimeOther +#define TensileHalf _Float16 +inline std::ostream& operator<<(std::ostream& os, const _Float16& dt) +{ + os << (float)(dt); + return os; +} + +template +struct tensile_complex +{ + t x; + t y; + + __host__ __device__ tensile_complex() = default; + + constexpr __host__ __device__ tensile_complex(t real, t imag = 0) + : x{real} + , y{imag} + { + } + + template {}>::type* = nullptr> + tensile_complex& operator=(const u a) + { + x = a; + y = 0; + return (*this); + } +}; + +template +inline std::ostream& operator<<(std::ostream& os, const tensile_complex& data) +{ + if(data.y >= 0) + { + os << data.x << "+" << data.y << "i"; + } + else + { + os << data.x << data.y << "i"; + } + + return os; +} + +template +constexpr __host__ __device__ tensile_complex operator+(tensile_complex const& data) +{ + return data; +} + +template +constexpr __host__ __device__ tensile_complex operator-(tensile_complex const& data) +{ + return tensile_complex{-data.x, -data.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator+(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x + b.x, a.y + b.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator-(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x - b.x, a.y - b.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator*(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x * b.x - a.y * b.y, a.x * b.y + a.y * b.x}; +} + +template +constexpr __host__ __device__ tensile_complex operator/(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{(a.x * b.x + a.y * b.y) / (b.x * b.x + b.y * b.y), + (a.y * b.x - a.x * b.y) / (b.x * b.x + b.y * b.y)}; +} + +template +constexpr __host__ __device__ tensile_complex& operator+=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a + b); +} + +template +constexpr __host__ __device__ tensile_complex& operator-=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a - b); +} + +template +constexpr __host__ __device__ tensile_complex& operator*=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a * b); +} + +template +constexpr __host__ __device__ tensile_complex& operator/=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a / b); +} +template +constexpr __host__ __device__ bool operator==(tensile_complex const& a, + tensile_complex const& b) +{ + return (a.x == b.x) && (a.y == b.y); +} + +using tensile_float_complex = tensile_complex; +using tensile_double_complex = tensile_complex; + +#define TensileComplexFloat tensile_float_complex +#define TensileComplexDouble tensile_double_complex +#endif // HIP + +#define TensileInt8x4 uint32_t +#define TensileInt32 int32_t + +/******************************************************************************* + * tensileSetup + ******************************************************************************/ +TensileStatus tensileSetup(); + +/******************************************************************************* + * tensileTeardown + ******************************************************************************/ +TensileStatus tensileTeardown(); + +/******************************************************************************* + * tensileCheckStatus + ******************************************************************************/ +#define tensileStatusCheck(RET) \ + { \ + TensileStatus tensileCheckStatusTmp = RET; \ + if(tensileCheckStatusTmp != tensileStatusSuccess) \ + { \ + fprintf(stderr, \ + "ERROR: TensileStatusFailure %i on line %u of %s\n", \ + tensileCheckStatusTmp, \ + __LINE__, \ + __FILE__); \ + abort(); \ + } \ + } + +// class ProblemDims +// - stores all dimensions of the problems (sizes and strides) +// TensileCreateLibrary.cpp will create a typedef for each specific problem, ie +// ProblemDims_Cijk_Ailk_Bljk_SB. +template +class ProblemDims +{ +public: + using SizeType = unsigned int; + + // Constructor accepts variable number of sizes: + template + explicit ProblemDims(Ts... args) + { + init(args...); + }; + + const SizeType strideD(int idx) const + { + return _dims[idx]; + } + + const SizeType strideC(int idx) const + { + return _dims[LastStrideD - FirstStride + idx]; + } + + const SizeType strideA(int idx) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + idx]; + } + + const SizeType strideB(int idx) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA + - FirstStride + idx]; + } + + const SizeType sizes(int idx = 0) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA + - FirstStride + LastStrideB - FirstStride + idx]; + } + + static int numSizes() + { + return NumSizes; + }; + + std::ostream& print(std::ostream& os) const + { + for(int i = 0; i < NumSizes; i++) + { + if(i != 0) + { + os << ", "; + }; + os << _dims[i]; + }; + return os; + }; + +private: + template + void init(T v) + { + _dims[NumSizes - I] = v; + } + + template + void init(T v, Ts... args) + { + _dims[NumSizes - I] = v; + init(args...); + } + +private: + // order: Dstride, Cstride, Astride, Bstrides, sizes + SizeType _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA - FirstStride + + LastStrideB - FirstStride + NumSizes]; +}; + +// Base template for ProblemKey +// - stores the sizes and strides +// - supports hash generation and comparison for lookup +// TensileCreateLibrary.cpp will create a typedef for each specific problem, ie +// ProblemKey_Cijk_Ailk_Bljk_SB. +// Some templates below use a parm called ProblemKeyType which can be any of +// these generated types. +template +class ProblemKey +{ +public: + using SizeType = unsigned int; + + // Constructor accepts variable number of sizes: + template + explicit ProblemKey(Ts... args) + : _db(0) + { + init(args...); + } + + template + explicit ProblemKey(const ProblemDims& pdims) + : _db(0) + , _equalStrides(true) + { + for(int i = 0; i < NumSizes; i++) + { + _sizes[i] = pdims.sizes(i); + } + + _equalStrides + = ((pdims.strideD(0) == pdims.strideC(0)) && (pdims.strideD(1) == pdims.strideC(1))); + } + + bool operator<(const ProblemKey& p) const + { + if(p._equalStrides != this->_equalStrides) + return true; + for(int i = 0; i < NumSizes; i++) + { + if(p._sizes[i] < this->_sizes[i]) + return true; + else if(p._sizes[i] > this->_sizes[i]) + return false; + // if equal, continue to next index + } + return false; // get here if all indices are equal + }; + + bool operator==(const ProblemKey& p) const + { + if(p._equalStrides != this->_equalStrides) + return false; + for(int i = 0; i < NumSizes; i++) + { + if(p._sizes[i] != this->_sizes[i]) + return false; + } + return true; + }; + + size_t hash() const + { + size_t h = 0; + for(int i = 0; i < NumSizes; i++) + { + h ^= _sizes[i] + 0x9b9773e99e3779b9 + (h << 6) + (h >> 2); + } + return h; + } + + const SizeType sizes(int i) const + { + return _sizes[i]; + }; + + static int numSizes() + { + return NumSizes; + }; + + std::ostream& print(std::ostream& os) const + { + for(int i = 0; i < NumSizes; i++) + { + if(i != 0) + { + os << ", "; + }; + os << _sizes[i]; + }; + return os; + }; + +private: + template + void init(const SizeType& v) + { + _sizes[NumSizes - I - 1] = v; + } + + template + void init(const SizeType& v, Ts... args) + { + _sizes[NumSizes - I - 1] = v; + init(args...); + } + +private: + // Data members: + SizeType _sizes[NumSizes]; + bool _equalStrides; + uint32_t _db; +}; + +//------------- +// Distance Functions between two problem sizes - used to find nearest neighbor +// or closest solution Assumes p1 and p2 have same number of sizes +//------------- + +template +struct RatioDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 1.0; + for(int i = 0; i < p1.numSizes(); i++) + { + distance += ::fabs(::log(double(p1.sizes(i)) / double(p2.sizes(i)))); + } + return distance; + } +}; + +template +struct ManhattanDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 0; + for(int i = 0; i < p1.numSizes(); i++) + { + distance += ::fabs(double(p1.sizes(i)) - double(p2.sizes(i))); + } + return distance; + } +}; + +template +struct EuclideanDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 0; + for(int i = 0; i < p1.numSizes(); i++) + { + if(p1.sizes(i) > p2.sizes(i)) + distance += pow(p1.sizes(i) - p2.sizes(i), 2); + else + distance += pow(p2.sizes(i) - p1.sizes(i), 2); + } + // distance = sqrt(distance); + return distance; + } +}; + +template +struct RandomDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + return double(rand()); + } +}; + +// ProblemType +// Stores the different dimensions (free, index, batch) and other +// problem-specific info A single ProblemType may have many ProblemDims +// Combination of a ProblemType and ProblemDim defines a Problem +class ProblemType +{ +public: + ProblemType(const std::vector& indicesFree, + const std::vector& indicesSummation, + const std::vector& indicesBatch, + const std::vector& indexAssignmentsA, + const std::vector& indexAssignmentsB) + : _indicesFree(indicesFree) + , _indicesSummation(indicesSummation) + , _indicesBatch(indicesBatch) + , _indexAssignmentsA(indexAssignmentsA) + , _indexAssignmentsB(indexAssignmentsB) + { + } + + unsigned lastSummationIdx() const + { + return _indicesSummation.back(); + }; + unsigned free0Idx() const + { + return _indicesFree[0]; + }; + unsigned free1Idx() const + { + return _indicesFree[1]; + }; + bool isBatchIdx(unsigned idx) const + { + return std::find(_indicesBatch.begin(), _indicesBatch.end(), idx) != _indicesBatch.end(); + }; + unsigned numIndicesC() const + { + return _indicesFree.size() + _indicesBatch.size(); + }; + + const std::vector indexAssignmentsA() const + { + return _indexAssignmentsA; + } + const std::vector indexAssignmentsB() const + { + return _indexAssignmentsB; + } + +private: + const std::vector _indicesFree; + const std::vector _indicesSummation; + const std::vector _indicesBatch; + const std::vector _indexAssignmentsA; + const std::vector _indexAssignmentsB; +}; + +// These are properties of the problemDims and problemType which drive the +// 'assertions' +// - Solutions may be optimized so they only work for a subset of +// ProblemProperties - +// - At runtime, the ProblemProperties are computed for the given ProblemType +// and +// ProblemDims, and then checked against the Solution requirements. +// Must be checked by the runtime before launching the solution +struct ProblemProperties +{ + + // Constructor used in solution tables- + // See writeSolutionAndExactTable in TensileCreateLibrary - this constructor + // must be in-sync with the table written there. + ProblemProperties(unsigned summationElementMultiple, + unsigned free0ElementMultiple, + unsigned free1ElementMultiple, + int approxSize, + bool equalStrides, + bool allBatchAStridesAreZero, + bool allBatchBStridesAreZero) + : _summationElementMultiple(summationElementMultiple) + , _free0ElementMultiple(free0ElementMultiple) + , _free1ElementMultiple(free1ElementMultiple) + , _approxSize(approxSize) + , _equalStrides(equalStrides) + , _allBatchAStridesAreZero(allBatchAStridesAreZero) + , _allBatchBStridesAreZero(allBatchBStridesAreZero) + , _db(0) + { + const char* db = std::getenv("TENSILE_DB"); + if(db) + { + _db = strtol(db, nullptr, 0); + } + } + + // Constructor used to compute assertions for a specified problem size + template + ProblemProperties(const ProblemDimsType& pdims, const ProblemType* props) + : _db(0) + { + _summationElementMultiple = 1; // problem summation element multiple + auto sumSize = pdims.sizes(props->lastSummationIdx()); + if((sumSize & 0x7) == 0) + _summationElementMultiple = 8; + else if((sumSize & 0x3) == 0) + _summationElementMultiple = 4; + else if((sumSize & 0x1) == 0) + _summationElementMultiple = 2; + + auto free0Size = pdims.sizes(props->free0Idx()); + _free0ElementMultiple = 1; // problem free0 element multiple + if((free0Size & 0x7) == 0) + _free0ElementMultiple = 8; + else if((free0Size & 0x3) == 0) + _free0ElementMultiple = 4; + else if((free0Size & 0x1) == 0) + _free0ElementMultiple = 2; + + auto free1Size = pdims.sizes(props->free1Idx()); + _free1ElementMultiple = 1; // problem free1 element multiple + if((free1Size & 0x7) == 0) + _free1ElementMultiple = 8; + else if((free1Size & 0x3) == 0) + _free1ElementMultiple = 4; + else if((free1Size & 0x1) == 0) + _free1ElementMultiple = 2; + + bool allBelow1 = true; + bool anyBelow1 = false; + bool anyBelow4 = false; + for(int si = 0; si != pdims.numSizes(); si++) + { + if(!props->isBatchIdx(si)) + { + auto size = pdims.sizes(si); + if(size > 1) + allBelow1 = false; + if(size == 1) + anyBelow1 = true; + else if(size < 4) + anyBelow4 = true; + } + } + if(allBelow1) + _approxSize = 1; // really small + else if(anyBelow1 || anyBelow4) + _approxSize = 2; // one dim not big enough + else + _approxSize = 99; // big enough + + _equalStrides = true; + // Would need to fix for UseInitialStrides, if UseInitialStrides==1 then + // don't subtract 1 + for(int i = 0; i < props->numIndicesC() - 1; i++) + { + if(pdims.strideD(i) != pdims.strideC(i)) + { + _equalStrides = false; + break; + } + } + + _allBatchAStridesAreZero = 1; + int strideIdx = 0; + for(auto& idx : props->indexAssignmentsA()) + { + bool isb = props->isBatchIdx(idx); + // Would need to fix for UseInitialStrides + auto stride = strideIdx == 0 ? 1 : pdims.strideA(strideIdx - 1); + if(isb && stride != 0) + { + _allBatchAStridesAreZero = 0; + } + strideIdx++; + } + + _allBatchBStridesAreZero = 1; + strideIdx = 0; + for(auto& idx : props->indexAssignmentsB()) + { + bool isb = props->isBatchIdx(idx); + // Would need to fix for UseInitialStrides + auto stride = strideIdx == 0 ? 1 : pdims.strideB(strideIdx - 1); + if(isb && stride != 0) + { + _allBatchBStridesAreZero = 0; + } + strideIdx++; + } + }; + + // Returns True if this AsssertionProperties meet the requirements for the + // specified soluition (this object represents the 'Problem') + bool validForSolution(const ProblemProperties& solutionRequirements) const + { + bool rv + = (this->_summationElementMultiple >= solutionRequirements._summationElementMultiple) + && (this->_free0ElementMultiple >= solutionRequirements._free0ElementMultiple) + && (this->_free1ElementMultiple >= solutionRequirements._free1ElementMultiple) + && ((this->_approxSize) >= solutionRequirements._approxSize) + && ((this->_equalStrides) == solutionRequirements._equalStrides) + && ((this->_allBatchAStridesAreZero) >= solutionRequirements._allBatchAStridesAreZero) + && ((this->_allBatchBStridesAreZero) + >= solutionRequirements._allBatchBStridesAreZero); + +#if 0 + if(this->_db & 0x10) + { + if(!rv) + { +#define check_print(name, oper) \ + if(!(this->name oper solutionRequirements.name)) \ + { \ + std::cout << #name << "failed: !(" << this->name << #oper << solutionRequirements.name \ + << ")" << std::endl; \ + } + + std::cout << "Assertion failed:" << std::endl; + check_print(_summationElementMultiple, >=); + check_print(_free0ElementMultiple, >=); + check_print(_free1ElementMultiple, >=); + check_print(_approxSize, >=); + check_print(_equalStrides, ==); + check_print(_allBatchAStridesAreZero, >=); + check_print(_allBatchBStridesAreZero, >=); +#undef check_print + } + else + { + std::cout << "Satisfied asserts!" << std::endl; + } + } +#endif + + return rv; + } + + unsigned _summationElementMultiple; + unsigned _free0ElementMultiple; + unsigned _free1ElementMultiple; + int _approxSize; + bool _equalStrides; + bool _allBatchAStridesAreZero; // for problems: true if all batchA strides are + // 0. Solutions: true if all batchA strides + // must be 0 to run the kernel. Prob,Soln: x,0: + // ok: 0,1: FAILED_ASSERT, 1,1: OK . P>=S + bool _allBatchBStridesAreZero; // true if all batchB strides are 0 + uint32_t _db; +}; + +#endif diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack+.hsaco b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack+.hsaco new file mode 100644 index 0000000000..312a818b7e Binary files /dev/null and b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack+.hsaco differ diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack-.hsaco b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack-.hsaco new file mode 100644 index 0000000000..1ddee53dc8 Binary files /dev/null and b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack-.hsaco differ diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary.yaml b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary.yaml new file mode 100644 index 0000000000..f6fb9340a7 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary.yaml @@ -0,0 +1,180 @@ +--- +library: + rows: + - library: {index: 0, type: Single} + predicate: + type: And + value: + - index: 0 + type: WorkspaceCheck + value: [4, 0, 1] + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: BiasDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: [3] + - {type: AmaxDCheck, value: false} + - type: TypesEqual + value: [Float, Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: Activation, value: None} + - {type: ActivationComputeType, value: Float} + - {type: ActivationNoGuard, value: false} + - {type: UseGradient, value: false} + - {type: UseBias, value: 0} + - {type: UseE, value: false} + - {type: DataTypeE, value: Float} + - {type: StridedBatched, value: true} + - {type: GroupedGemm, value: false} + - {type: UseScaleAB, value: ''} + - {type: UseScaleCD, value: false} + - {type: UseScaleAlphaVec, value: 0} + - {type: Sparse, value: 0} + - {type: F32XdlMathOp, value: XFloat32} + - {type: SupportDeviceUserArguments, value: true} + - {type: SwizzleTensorA, value: false} + - {type: SwizzleTensorB, value: false} + - index: 0 + type: WorkgroupNumberCheck + value: [16, 16, 1] + - {type: LeadingFree0SizesGreaterOrEqual, value: 8} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: KernelLanguageCompatible, value: Assembly} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 32, ShiftPtrElemA: 8, ShiftPtrElemB: 4} + - {type: BufferLoadOffsetLimitCheck_Beta, value: 16} + - {type: BufferStoreOffsetLimitCheck, value: 16} + - type: GlobalSplitUCheckMinK + value: [32, 1] + - type: WorkgroupMappingXCCCheck + value: [1, -1] + type: Problem +solutions: +- debugKernel: false + hardwarePredicate: + type: AMDGPU + value: {type: Processor, value: gfx950} + ideals: {} + index: 0 + internalArgsSupport: {gsu: true, staggerU: true, useUniversalArgs: true, version: 2, + wgm: true} + kernelName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1 + libraryLogicIndex: -1 + linearModel: {} + name: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 + problemPredicate: + type: And + value: + - index: 0 + type: WorkspaceCheck + value: [4, 0, 1] + - {type: OperationIdentifierEqual, value: Contraction_l_Ailk_Bjlk_Cijk_Dijk} + - type: BiasDataTypeWhiteList + value: [] + - type: BiasSrcWhiteList + value: [3] + - {type: AmaxDCheck, value: false} + - type: TypesEqual + value: [Float, Float, Float, Float, Float] + - {type: HighPrecisionAccumulate, value: false} + - {type: Activation, value: None} + - {type: ActivationComputeType, value: Float} + - {type: ActivationNoGuard, value: false} + - {type: UseGradient, value: false} + - {type: UseBias, value: 0} + - {type: UseE, value: false} + - {type: DataTypeE, value: Float} + - {type: StridedBatched, value: true} + - {type: GroupedGemm, value: false} + - {type: UseScaleAB, value: ''} + - {type: UseScaleCD, value: false} + - {type: UseScaleAlphaVec, value: 0} + - {type: Sparse, value: 0} + - {type: F32XdlMathOp, value: XFloat32} + - {type: SupportDeviceUserArguments, value: true} + - {type: SwizzleTensorA, value: false} + - {type: SwizzleTensorB, value: false} + - index: 0 + type: WorkgroupNumberCheck + value: [16, 16, 1] + - {type: LeadingFree0SizesGreaterOrEqual, value: 8} + - {type: LeadingFree1SizesGreaterOrEqual, value: 4} + - {type: KernelLanguageCompatible, value: Assembly} + - type: BufferLoadOffsetLimitCheck + value: {DUorMT0: 32, DUorMT1: 32, ShiftPtrElemA: 8, ShiftPtrElemB: 4} + - {type: BufferLoadOffsetLimitCheck_Beta, value: 16} + - {type: BufferStoreOffsetLimitCheck, value: 16} + - type: GlobalSplitUCheckMinK + value: [32, 1] + - type: WorkgroupMappingXCCCheck + value: [1, -1] + problemType: + aType: Float + activationArgLength: 0 + activationComputeDataType: Float + activationNoGuard: false + activationType: None + bType: Float + biasDataTypeWhiteList: [] + biasSrcWhiteList: [3] + cType: Float + computeInputType: Float + computeType: Float + dType: Float + eType: Float + f32XdlMathOp: XFloat32 + groupedGemm: false + highPrecisionAccumulate: false + operationIdentifier: Contraction_l_Ailk_Bjlk_Cijk_Dijk + outputAmaxD: false + sparse: 0 + stridedBatched: true + supportDeviceUserArguments: true + swizzleTensorA: false + swizzleTensorB: false + transA: false + transB: true + useBeta: true + useBias: 0 + useE: false + useGradient: false + useInitialStridesAB: false + useInitialStridesCD: false + useScaleAB: '' + useScaleAlphaVec: 0 + useScaleCD: false + sizeMapping: + CUOccupancy: 5 + CustomKernelName: '' + MathClocksUnrolledLoop: 109 + PrefetchGlobalRead: 1 + activationFused: true + depthU: 32 + globalAccumulation: 2 + globalSplitU: 1 + globalSplitUCoalesced: false + globalSplitUPGR: 16 + globalSplitUWorkGroupMappingRoundRobin: false + grvwA: 8 + grvwB: 4 + gwvwC: 1 + gwvwD: 1 + macroTile: [16, 16, 1] + magicDivAlg: 2 + matrixInstruction: [16, 16, 32, 1] + packBatchDims: 0 + sourceKernel: false + staggerStrideShift: 1 + staggerU: 32 + staggerUMapping: 0 + streamK: 0 + streamKAtomic: 0 + threadTile: [1, 1] + waveNum: 1 + workGroup: [16, 4, 1] + workGroupMapping: 8 + workGroupMappingXCC: 1 + workGroupMappingXCCGroup: -1 + workspaceSizePerElemBias: 0 + workspaceSizePerElemC: 4 +... diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary_gfx950.co b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary_gfx950.co new file mode 100755 index 0000000000..538a27f362 Binary files /dev/null and b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary_gfx950.co differ diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h new file mode 100644 index 0000000000..b615e09981 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h @@ -0,0 +1,597 @@ +/****************************************************************************** + * Copyright (C) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + *****************************************************************************/ +/*! \file + \brief Architecture-specific operators on memory added for GFX9 +*/ +// reference: +// https://github.com/llvm/llvm-project/blob/main/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll + +#ifndef INTRINSIC_MEM_ACCESS_H +#define INTRINSIC_MEM_ACCESS_H + +#if defined(__clang__) && defined(__HIP__) + +#if(defined(__NVCC__) || defined(__HIPCC__)) \ + || (defined(__clang__) && (defined(__CUDA__)) || defined(__HIP__)) +#define INLINEDEVICE __forceinline__ __device__ +#elif defined(__CUDACC_RTC__) +#define INLINEDEVICE __forceinline__ __device__ +#else +#define INLINEDEVICE inline +#endif + +#if defined(__gfx803__) || defined(__gfx900__) || defined(__gfx906__) || defined(__gfx908__) \ + || defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) // test device +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x00020000 +#elif defined(__gfx1030__) // special device +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x31014000 +#elif defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) \ + || defined(__gfx1150__) || defined(__gfx1151__) +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x31004000 +#elif defined(__gfx1200__) || defined(__gfx1201__) +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x30020000 +#else // not support +#define BUFFER_RESOURCE_3RD_DWORD -1 +#endif + +/// Controls AMD gfx arch cache operations +struct CacheOperation +{ + enum Kind + { + /// Cache at all levels - accessed again + Always, + /// Cache at global level; glc = 1 + Global, + /// Streaming - likely to be accessed once; slc = 1 + Streaming, + /// Indicates the line will not be used again, glc = 1; slc = 1 + LastUse + }; +}; + +using float16_t = _Float16; +using float32_t = float; + +template +struct NativeVector +{ + using type = T __attribute__((ext_vector_type(N))); +}; + +using float32x2_t = NativeVector::type; +using float32x4_t = NativeVector::type; + +using int32x4_t = NativeVector::type; +using int32x2_t = NativeVector::type; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct alignas(16) BufferResource +{ + union Desc + { + int32x4_t d128; + void* d64[2]; + uint32_t d32[4]; + }; + + INLINEDEVICE + BufferResource(void const* base_addr, uint32_t num_records = (0xFFFFFFFF - 1)) + { + // Reference: + // For CDNA: see section 9.1.8 in the AMD resources + // https://developer.amd.com/wp-content/resources/CDNA1_Shader_ISA_14December2020.pdf + // For RDNA: see section 8.1.8 in the AMD resources + // https://developer.amd.com/wp-content/resources/RDNA2_Shader_ISA_November2020.pdf + // The d32[3] field represents the 0x[127] ~ [96] + + // 64-bit base address + desc_.d64[0] = const_cast(base_addr); + // 32-bit number of records in bytes which is used to guard against out-of-range access + desc_.d32[2] = num_records; + // 32-bit buffer resource descriptor + desc_.d32[3] = BUFFER_RESOURCE_3RD_DWORD; + } + + INLINEDEVICE + operator int32x4_t() + { + // return desc_.d128; // NOTE HIP: Crashes compiler; see below + + /// This hack is to enforce scalarization of the variable "base_addr", where in some + /// circumstances it becomes vectorized and then in turn causes illegal lowering to GCN ISA + /// since compiler effectively tries to stuff VGPRs in slots where it only accepts SGPRs + Desc ret; + ret.d32[0] = __builtin_amdgcn_readfirstlane(desc_.d32[0]); + ret.d32[1] = __builtin_amdgcn_readfirstlane(desc_.d32[1]); + ret.d64[1] = desc_.d64[1]; + return ret.d128; + /// + } + + Desc desc_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// +/// Load +/// + +// 1 byte +__device__ char + llvm_amdgcn_raw_buffer_load_i8(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.i8"); + +// 2 bytes +__device__ float16_t + llvm_amdgcn_raw_buffer_load_f16(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.f16"); + +// 4 bytes +__device__ float32_t + llvm_amdgcn_raw_buffer_load_f32(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.f32"); + +// 8 bytes +__device__ float32x2_t + llvm_amdgcn_raw_buffer_load_f32x2(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v2f32"); + +// 16 bytes +__device__ float32x4_t + llvm_amdgcn_raw_buffer_load_f32x4(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v4f32"); + +// 4 bytes +__device__ int32_t + llvm_amdgcn_s_buffer_load_i32(int32x4_t buffer_resource, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.i32"); + +// 8 bytes +__device__ int32x2_t + llvm_amdgcn_s_buffer_load_i32x2(int32x4_t buffer_resource, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.v2i32"); + +// 16 bytes +__device__ int32x4_t + llvm_amdgcn_s_buffer_load_i32x4(int32x4_t buffer_resource, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.v4i32"); + +/// +/// Store +/// + +// 1 byte +__device__ void + llvm_amdgcn_raw_buffer_store_i8(char data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.i8"); + +// 2 bytes +__device__ void + llvm_amdgcn_raw_buffer_store_f16(float16_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.f16"); + +// 4 bytes +__device__ void + llvm_amdgcn_raw_buffer_store_f32(float32_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.f32"); + +// 8 bytes +__device__ void llvm_amdgcn_raw_buffer_store_f32x2( + float32x2_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.v2f32"); + +// 16 bytes +__device__ void llvm_amdgcn_raw_buffer_store_f32x4( + float32x4_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.v4f32"); + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// Fragment type to store loaded data + typename AccessType, + /// The bytes of loading + int LoadBytes, + /// Cache operation + CacheOperation::Kind cache_op = CacheOperation::Always> +struct buffer_load; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + char ret = llvm_amdgcn_raw_buffer_load_i8( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + char ret = llvm_amdgcn_raw_buffer_load_i8( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float16_t ret = llvm_amdgcn_raw_buffer_load_f16( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float16_t ret = llvm_amdgcn_raw_buffer_load_f16( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32_t ret = llvm_amdgcn_raw_buffer_load_f32( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32_t ret = llvm_amdgcn_raw_buffer_load_f32( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x2_t ret = llvm_amdgcn_raw_buffer_load_f32x2( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x2_t ret = llvm_amdgcn_raw_buffer_load_f32x2( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x4_t ret = llvm_amdgcn_raw_buffer_load_f32x4( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x4_t ret = llvm_amdgcn_raw_buffer_load_f32x4( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template < + /// Fragment type to store loaded data + typename AccessType, + /// The bytes of loading + int LoadBytes, + /// Cache operation + CacheOperation::Kind cache_op = CacheOperation::Always> +struct s_buffer_load; + +template +struct s_buffer_load +{ + INLINEDEVICE + s_buffer_load() {} + + INLINEDEVICE + s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32_t ret = llvm_amdgcn_s_buffer_load_i32( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32_t ret = llvm_amdgcn_s_buffer_load_i32( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct s_buffer_load +{ + INLINEDEVICE + s_buffer_load() {} + + INLINEDEVICE + s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x2_t ret = llvm_amdgcn_s_buffer_load_i32x2( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x2_t ret = llvm_amdgcn_s_buffer_load_i32x2( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct s_buffer_load +{ + INLINEDEVICE + s_buffer_load() {} + + INLINEDEVICE + s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x4_t ret = llvm_amdgcn_s_buffer_load_i32x4( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x4_t ret = llvm_amdgcn_s_buffer_load_i32x4( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// Fragment type to store loaded data + typename AccessType, + /// The width of loading + int NumElements, + /// Cache operation + CacheOperation::Kind cache_op = CacheOperation::Always> +struct buffer_store; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + char data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_i8( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float16_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f16( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f32( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x2_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f32x2( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x4_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f32x4( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#endif // defined(__clang__) && defined(__HIP__) + +#endif // INTRINSIC_MEM_ACCESS_H diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h new file mode 100644 index 0000000000..b059db2109 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h @@ -0,0 +1,251 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +/*!\file + * \brief tensile_bfloat16.h provides struct for tensile_bfloat16 typedef + */ + +#pragma once +#ifndef _TENSILE_BFLOAT16_H_ +#define _TENSILE_BFLOAT16_H_ + +// If this is a C compiler, C++ compiler below C++11, or a host-only compiler, +// we only include a minimal definition of tensile_bfloat16 +#if __cplusplus < 201103L || !defined(__HIPCC__) + +#include +typedef struct +{ + uint16_t data; +} tensile_bfloat16; + +#else // __cplusplus < 201103L || !defined(__HIPCC__) + +#include +#include +#include +#include +#include +#include + +struct tensile_bfloat16 +{ + uint16_t data; + + // Don't initialize `data` in purpose so that it could be used with + // `__shared__`, which forbid any initializer. + __host__ __device__ tensile_bfloat16() {} + + // round upper 16 bits of IEEE float to convert to bfloat16 + explicit __host__ __device__ tensile_bfloat16(float f) + : data(float_to_bfloat16(f)) + { + } + + // zero extend lower 16 bits of bfloat16 to convert to IEEE float + __host__ __device__ operator float() const + { + union + { + uint32_t int32; + float fp32; + } u = {uint32_t(data) << 16}; + return u.fp32; + } + +private: + static __host__ __device__ uint16_t float_to_bfloat16(float f) + { + union + { + float fp32; + uint32_t int32; + } u = {f}; + if(~u.int32 & 0x7f800000) + { + // When the exponent bits are not all 1s, then the value is zero, normal, + // or subnormal. We round the bfloat16 mantissa up by adding 0x7FFF, plus + // 1 if the least significant bit of the bfloat16 mantissa is 1 (odd). + // This causes the bfloat16's mantissa to be incremented by 1 if the 16 + // least significant bits of the float mantissa are greater than 0x8000, + // or if they are equal to 0x8000 and the least significant bit of the + // bfloat16 mantissa is 1 (odd). This causes it to be rounded to even when + // the lower 16 bits are exactly 0x8000. If the bfloat16 mantissa already + // has the value 0x7f, then incrementing it causes it to become 0x00 and + // the exponent is incremented by one, which is the next higher FP value + // to the unrounded bfloat16 value. When the bfloat16 value is subnormal + // with an exponent of 0x00 and a mantissa of 0x7F, it may be rounded up + // to a normal value with an exponent of 0x01 and a mantissa of 0x00. + // When the bfloat16 value has an exponent of 0xFE and a mantissa of 0x7F, + // incrementing it causes it to become an exponent of 0xFF and a mantissa + // of 0x00, which is Inf, the next higher value to the unrounded value. + u.int32 += 0x7fff + ((u.int32 >> 16) & 1); // Round to nearest, round to even + } + else if(u.int32 & 0xffff) + { + // When all of the exponent bits are 1, the value is Inf or NaN. + // Inf is indicated by a zero mantissa. NaN is indicated by any nonzero + // mantissa bit. Quiet NaN is indicated by the most significant mantissa + // bit being 1. Signaling NaN is indicated by the most significant + // mantissa bit being 0 but some other bit(s) being 1. If any of the + // lower 16 bits of the mantissa are 1, we set the least significant bit + // of the bfloat16 mantissa, in order to preserve signaling NaN in case + // the bloat16's mantissa bits are all 0. + u.int32 |= 0x10000; // Preserve signaling NaN + } + return uint16_t(u.int32 >> 16); + } +}; + +static_assert(std::is_standard_layout{}, + "tensile_bfloat16 is not a standard layout type, and thus is " + "incompatible with C."); + +static_assert(std::is_trivially_copyable{}, + "tensile_bfloat16 is not trivially copyable, and thus is " + "incompatible with C."); + +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat16& bf16) +{ + return os << float(bf16); +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a) +{ + return a; +} +inline __host__ __device__ tensile_bfloat16 operator-(tensile_bfloat16 a) +{ + a.data ^= 0x8000; + return a; +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) + float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator+(int a, tensile_bfloat16 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a, int b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline __host__ __device__ tensile_bfloat16 operator-(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) - float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator*(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) * float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator/(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) / float(b)); +} +inline __host__ __device__ bool operator<(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return float(a) < float(b); +} +inline __host__ __device__ bool operator==(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return float(a) == float(b); +} +inline __host__ __device__ bool operator>(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return b < a; +} +inline __host__ __device__ bool operator<=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a > b); +} +inline __host__ __device__ bool operator!=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a == b); +} +inline __host__ __device__ bool operator>=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a < b); +} +inline __host__ __device__ tensile_bfloat16& operator+=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a + b; +} +inline __host__ __device__ tensile_bfloat16& operator-=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a - b; +} +inline __host__ __device__ tensile_bfloat16& operator*=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a * b; +} +inline __host__ __device__ tensile_bfloat16& operator/=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a / b; +} +inline __host__ __device__ tensile_bfloat16& operator++(tensile_bfloat16& a) +{ + return a += tensile_bfloat16(1.0f); +} +inline __host__ __device__ tensile_bfloat16& operator--(tensile_bfloat16& a) +{ + return a -= tensile_bfloat16(1.0f); +} +inline __host__ __device__ tensile_bfloat16 operator++(tensile_bfloat16& a, int) +{ + tensile_bfloat16 orig = a; + ++a; + return orig; +} +inline __host__ __device__ tensile_bfloat16 operator--(tensile_bfloat16& a, int) +{ + tensile_bfloat16 orig = a; + --a; + return orig; +} +inline __host__ __device__ bool isinf(tensile_bfloat16 a) +{ + return !(~a.data & 0x7f80) && !(a.data & 0x7f); +} +inline __host__ __device__ bool isnan(tensile_bfloat16 a) +{ + return !(~a.data & 0x7f80) && +(a.data & 0x7f); +} +inline __host__ __device__ bool iszero(tensile_bfloat16 a) +{ + return !(a.data & 0x7fff); +} +inline tensile_bfloat16 sin(tensile_bfloat16 a) +{ + return tensile_bfloat16(sinf(float(a))); +} +inline tensile_bfloat16 cos(tensile_bfloat16 a) +{ + return tensile_bfloat16(cosf(float(a))); +} + +#endif // __cplusplus < 201103L || !defined(__HIPCC__) + +#endif // _TENSILE_BFLOAT16_H_ diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h new file mode 100644 index 0000000000..fe7b92ba37 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h @@ -0,0 +1,650 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2019-2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#ifdef TENSILE_USE_HIP +#include +#endif + +// comment out following macro to disable FP8/BF8 types +#define TENSILE_USE_FP8_BF8 + +#define HIP_HOST_DEVICE __host__ __device__ +#define HIP_HOST __host__ +#define HIP_DEVICE __device__ + +//namespace Tensile +//{ + + struct tensile_hip_fp8_e4m3: public __hip_fp8_e4m3 + { + using __hip_fp8_e4m3:: __hip_fp8_e4m3; // list base's constructor in derive's scope + + // constructor -> down cast +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE tensile_hip_fp8_e4m3(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e4m3(const _Float16 f) +#endif + : __hip_fp8_e4m3(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const float& a) + { + __x = tensile_hip_fp8_e4m3(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const double& a) + { + __x = tensile_hip_fp8_e4m3((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const tensile_hip_fp8_e4m3& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e4m3(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator+=(const tensile_hip_fp8_e4m3& a) + { + __x = tensile_hip_fp8_e4m3(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + + + typedef tensile_hip_fp8_e4m3 tensile_float8; + + struct tensile_hip_fp8_e5m2: public __hip_fp8_e5m2 + { + + using __hip_fp8_e5m2:: __hip_fp8_e5m2; + +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE tensile_hip_fp8_e5m2(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e5m2(const _Float16 f) +#endif + : __hip_fp8_e5m2(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const float& a) + { + __x = tensile_hip_fp8_e5m2(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const double& a) + { + __x = tensile_hip_fp8_e5m2((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const tensile_hip_fp8_e5m2& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e5m2(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator+=(const tensile_hip_fp8_e5m2& a) + { + __x = tensile_hip_fp8_e5m2(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + + + typedef tensile_hip_fp8_e5m2 tensile_bfloat8; + + struct tensile_hip_fp8_e4m3_fnuz: public __hip_fp8_e4m3_fnuz + { + using __hip_fp8_e4m3_fnuz:: __hip_fp8_e4m3_fnuz; + +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e4m3_fnuz(const _Float16 f) +#endif + : __hip_fp8_e4m3_fnuz(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const float& a) + { + __x = tensile_hip_fp8_e4m3_fnuz(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const double& a) + { + __x = tensile_hip_fp8_e4m3_fnuz((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const tensile_hip_fp8_e4m3_fnuz& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e4m3_fnuz(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator+=(const tensile_hip_fp8_e4m3_fnuz& a) + { + __x = tensile_hip_fp8_e4m3_fnuz(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + typedef tensile_hip_fp8_e4m3_fnuz tensile_float8_fnuz; + + + struct tensile_hip_fp8_e5m2_fnuz: public __hip_fp8_e5m2_fnuz + { + using __hip_fp8_e5m2_fnuz:: __hip_fp8_e5m2_fnuz; + +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e5m2_fnuz(const _Float16 f) +#endif + : __hip_fp8_e5m2_fnuz(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const float& a) + { + __x = tensile_hip_fp8_e5m2_fnuz(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const double& a) + { + __x = tensile_hip_fp8_e5m2_fnuz((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const tensile_hip_fp8_e5m2_fnuz& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e5m2_fnuz(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator+=(const tensile_hip_fp8_e5m2_fnuz& a) + { + __x = tensile_hip_fp8_e5m2_fnuz(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + typedef tensile_hip_fp8_e5m2_fnuz tensile_bfloat8_fnuz; + + + // Other operator overloading + inline std::ostream& operator<<(std::ostream& os, const tensile_float8& f8) + { + os << static_cast(f8); + return os; + } + inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8& bf8) + { + os << static_cast(bf8); + return os; + } + inline std::ostream& operator<<(std::ostream& os, const tensile_float8_fnuz& f8) + { + os << static_cast(f8); + return os; + } + inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8_fnuz& bf8) + { + os << static_cast(bf8); + return os; + } + + // + inline tensile_float8 operator+(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8 operator+(tensile_float8 a, float b) + { + return static_cast(static_cast(a) + b); + } + inline tensile_float8 operator+(float a, tensile_float8 b) + { + return static_cast(a + static_cast(b)); + } + inline tensile_bfloat8 operator+(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8 operator-(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + inline tensile_bfloat8 operator-(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + // NOTE: It is not used in reference solution directly, we want to return float otherwise + inline tensile_float8 operator*(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + inline tensile_float8 operator*(float a, tensile_float8 b) + { + return static_cast(a * static_cast(b)); + } + inline tensile_float8 operator*(tensile_float8 a, float b) + { + return static_cast(static_cast(a) * b); + } + inline tensile_bfloat8 operator*(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + + inline tensile_float8 operator/(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline tensile_bfloat8 operator/(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline bool operator<(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<(float a, tensile_float8 b) + { + return a < static_cast(b); + } + inline bool operator<(tensile_float8 a, float b) + { + return static_cast(a) < b; + } + inline bool operator<(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<=(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator<=(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator==(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator==(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator!=(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator!=(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator>(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>(float a, tensile_float8 b) + { + return a > static_cast(b); + } + inline bool operator>(tensile_float8 a, float b) + { + return static_cast(a) > b; + } + inline bool operator>(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>=(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) >= static_cast(b); + } + inline bool operator>=(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) >= static_cast(b); + } + +// FNUZ + inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, float b) + { + return static_cast(static_cast(a) + b); + } + inline tensile_float8_fnuz operator+(float a, tensile_float8_fnuz b) + { + return static_cast(a + static_cast(b)); + } + inline tensile_bfloat8_fnuz operator+(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8_fnuz operator-(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + inline tensile_bfloat8_fnuz operator-(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + // NOTE: It is not used in reference solution directly, we want to return float otherwise + inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + inline tensile_float8_fnuz operator*(float a, tensile_float8_fnuz b) + { + return static_cast(a * static_cast(b)); + } + inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, float b) + { + return static_cast(static_cast(a) * b); + } + inline tensile_bfloat8_fnuz operator*(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + + inline tensile_float8_fnuz operator/(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline tensile_bfloat8_fnuz operator/(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline bool operator<(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<(float a, tensile_float8_fnuz b) + { + return a < static_cast(b); + } + inline bool operator<(tensile_float8_fnuz a, float b) + { + return static_cast(a) < b; + } + inline bool operator<(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<=(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator<=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator==(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator==(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator!=(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator!=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator>(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>(float a, tensile_float8_fnuz b) + { + return a > static_cast(b); + } + inline bool operator>(tensile_float8_fnuz a, float b) + { + return static_cast(a) > b; + } + inline bool operator>(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>=(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) >= static_cast(b); + } + inline bool operator>=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) >= static_cast(b); + } +//} // end of namespace Tensile + +namespace std +{ + inline bool isinf(const tensile_float8& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8& a) + { + return (static_cast(a) & 0x7f) == 0x7c; + } + + inline bool isnan(const tensile_float8& a) + { + return (static_cast(a) & 0x7f) == 0x7f; + } + inline bool isnan(const tensile_bfloat8& a) + { + return (static_cast(a) & 0x7f) > 0x7c; + } + inline bool iszero(const tensile_float8& a) + { + return (static_cast(a) & 0x7F) == 0x0; + } + inline bool iszero(const tensile_bfloat8& a) + { + return (static_cast(a) & 0x7F) == 0x0; + } + + //TODO: better to & 0x7F + inline tensile_float8 abs(const tensile_float8& a) + { + return tensile_float8(std::abs(float(a))); + } + inline tensile_bfloat8 abs(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::abs(float(a))); + } + + inline tensile_float8 sin(const tensile_float8& a) + { + return tensile_float8(std::sin(float(a))); + } + inline tensile_bfloat8 sin(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::sin(float(a))); + } + + inline tensile_float8 cos(const tensile_float8& a) + { + return tensile_float8(std::cos(float(a))); + } + inline tensile_bfloat8 cos(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8& a) + { + return std::to_string(static_cast(a)); + } + // FNUZ + inline bool isinf(const tensile_float8_fnuz& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8_fnuz& a) + { + return false; + } + + inline bool isnan(const tensile_float8_fnuz& a) + { + return static_cast(a) == 0x80; + } + inline bool isnan(const tensile_bfloat8_fnuz& a) + { + return static_cast(a) == 0x80; + } + inline bool iszero(const tensile_float8_fnuz& a) + { + return static_cast(a) == 0x0; // NOTE: only +0 exists + } + inline bool iszero(const tensile_bfloat8_fnuz& a) + { + return static_cast(a) == 0x0; // NOTE: only +0 exists + } + + inline tensile_float8_fnuz abs(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::abs(float(a))); + } + inline tensile_bfloat8_fnuz abs(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::abs(float(a))); + } + + inline tensile_float8_fnuz sin(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::sin(float(a))); + } + inline tensile_bfloat8_fnuz sin(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::sin(float(a))); + } + + inline tensile_float8_fnuz cos(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::cos(float(a))); + } + inline tensile_bfloat8_fnuz cos(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + +} // namespace std diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h new file mode 100644 index 0000000000..4c32c0086e --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h @@ -0,0 +1,1215 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2019-2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#define HIP_HOST_DEVICE __host__ __device__ +#define HIP_HOST __host__ +#define HIP_DEVICE __device__ + +// We are clipping in down--conversion by default +#define DOWNCAST_CLIPPING_ON 1 + +#ifdef DOWNCAST_CLIPPING_ON +constexpr bool downcast_clip_on = true; +#else +constexpr bool downcast_clip_on = false; +#endif + +// TODO: comment about ocp not used, just filler, will replace with hip fp8 header + +namespace tensile_hip_f8_impl +{ + __host__ inline int clz(uint32_t x) + { + return __builtin_clz(x); + } + __device__ inline int clz(uint32_t x) + { + return __clz(x); + } + + template + HIP_HOST_DEVICE uint8_t cast_to_f8(T _x, bool stoch, uint32_t rng) + { + constexpr bool is_half = std::is_same::value; + constexpr bool is_float = std::is_same::value; + static_assert(wm + we == 7, "wm+we==7"); + static_assert(is_half || is_float, "Only half and float can be cast to f8"); + + //if(sizeof(T)==2 && we==5 && !negative_zero_nan) + //return cast_to_f8_no_range_reduce<2, 5, _Float16>(_x, stoch, rng); + + const int mfmt = (sizeof(T) == 4) ? 23 : 10; + uint32_t x; + if(sizeof(T) == 4) + x = reinterpret_cast(_x); + else + x = reinterpret_cast(_x); + + uint32_t y, head, mantissa; + int exponent, bias; + uint32_t sign; + + if(sizeof(T) == 4) + { + head = x & 0xFF800000; + mantissa = x & 0x7FFFFF; + exponent = (head >> 23) & 0xFF; + sign = head >> 31; + bias = 127; + } + else + { + head = x & 0xFC00; + mantissa = x & 0x3FF; + exponent = (head >> 10) & 0x1F; + sign = head >> 15; + bias = 15; + } + + uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm); + + // Deal with inf and NaNs + if(negative_zero_nan) + { + if(sizeof(T) == 4) + { + if((x & 0x7F800000) == 0x7F800000) + return 0x80; + } + else + { + //if(__hisinf(x) || __hisnan(x)) + if((x & 0x7C00) == 0x7C00) + return 0x80; + } + } + else + { + if(sizeof(T) == 4) + { + if((x & 0x7F800000) == 0x7F800000) + return signed_inf + (mantissa != 0 ? 1 : 0); + } + else + { + if((x & 0x7C00) == 0x7C00) + return signed_inf + (mantissa != 0 ? 1 : 0); + } + } + if(x == 0) + return 0; + + // First need to check if it is normal or denorm as there is a difference of implict 1 + // Then need to adjust the exponent to align with the F8 exponent, in the meanwhile, shift + // The mantissa. Then for stochastic rounding, add rng to mantissa and truncate. And for + // RNE, no need to add rng. Then probably need to check whether there is carry and adjust + // exponent and mantissa again + + // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent bits + const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0); + const int f8_denormal_act_exponent = 1 - f8_bias; //actual exponent of f8 denormal + // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias) + // f8_exponent is the converted f8 exponent with bias encoding + // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent, + // the difference needs to be adjusted and mantissa shifted + int act_exponent, f8_exponent, exponent_diff; + + if(exponent == 0) + { // fp32/fp16 is in denormal. + /* fp32 denormal is below 2^-127 so it is usually not a concern here, we mostly concern fp16 here. + In this case, f8 is usually in denormal. But there could be exceptions. + fp16 denormal has exponent bias 15 while bf8 with NANOO has exponent bias 16. + It means that there are some numbers in fp16 denormal but they are bf8 (NANOO) normals - smallest bf8 (NANOO) normal is 2^-15. + fp16 numbers where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 are bf8 (NANOO) normal. + In this case, the fp16 mantissa should be shift left by 1 */ + act_exponent = exponent - bias + 1; + exponent_diff = f8_denormal_act_exponent + - act_exponent; // actual exponent is exponent-bias+1 as it is denormal + } + else + { // fp32/fp16 is normal with implicit 1 + act_exponent = exponent - bias; + if(act_exponent <= f8_denormal_act_exponent) + { + /* This is the case where fp32/fp16 is normal but it is in f8 denormal range. + For example fp8 nanoo mode, denormal exponent is -7, but if the fp32/fp16 + actual exponent is -7, it is actually larger due to the implict 1, + Therefore it needs to be adjust to -6 and mantissa shift right by 1. + So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */ + exponent_diff = f8_denormal_act_exponent - act_exponent; + } + else + { //both fp32/fp16 and f8 are in normal range + exponent_diff + = 0; // exponent_diff=0 does not mean there is no difference for this case, + //act_exponent could be larger. Just that it does not need shift mantissa + } + mantissa += (1 << mfmt); //Add the implicit 1 into mantissa + } + + bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) + == (1 << (mfmt - wm + exponent_diff - 1)); + /* This part is a bit tricky. The judgment of whether it is a tie needs to be done before we shift right + as shift right could rip off some residual part and make something not midpoint look like midpoint. + For example, the fp16 number 0x1002 (0 00100 0000000010), it is larger than midpoint, + but after shift right by 4 bits, it would look like midpoint. + */ + + if(exponent_diff > 0) + mantissa >>= exponent_diff; + else if(exponent_diff == -1) + mantissa <<= -exponent_diff; + bool implicit_one = mantissa & (1 << mfmt); + //if there is no implict 1, it means the f8 is denormal and need to adjust to denorm exponent + f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias + - (implicit_one ? 0 : 1); + + //Now we have the exponent and mantissa adjusted + uint32_t drop_mask = (1 << (mfmt - wm)) - 1; + //bool midpoint = (mantissa & drop_mask) == ( 1 << (mfmt-wm-1) ); + bool odd = mantissa + & (1 << (mfmt - wm)); // if the least significant bit that is not truncated is 1 + mantissa + += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask; + + //Now we deal with overflow + if(f8_exponent == 0) + { + if((1 << mfmt) & mantissa) + { + f8_exponent = 1; //denormal overflow to become normal, promote exponent + //mantissa &= (1<>= 1; + f8_exponent++; + //mantissa &= (1<>= (mfmt - wm); + + // above range: quantize to maximum possible float of the same sign + const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2); + if(f8_exponent > max_exp) + { + if(clip) + { + mantissa = (1 << wm) - 1; + f8_exponent = max_exp; + } + else + { + return signed_inf; + } + } + + if(f8_exponent == 0 && mantissa == 0) + return negative_zero_nan ? 0 : (sign << 7); + mantissa &= (1 << wm) - 1; + return (sign << 7) | (f8_exponent << wm) | mantissa; + } + + template + HIP_HOST_DEVICE T cast_from_f8(uint8_t x) + { + constexpr bool is_half = std::is_same::value; + constexpr bool is_float = std::is_same::value; + //constexpr bool is_bf16 = std::is_same::value; + static_assert(is_half || is_float, "only half and float are supported"); + + constexpr int weo = is_half ? 5 : 8; + constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7); + + T fInf, fNegInf, fNaN, fNeg0; + if(is_half) + { + const uint16_t ihInf = 0x7C00; + const uint16_t ihNegInf = 0xFC00; + const uint16_t ihNaN = 0x7C01; + const uint16_t ihNeg0 = 0x8000; + fInf = reinterpret_cast(ihInf); + fNegInf = reinterpret_cast(ihNegInf); + fNaN = reinterpret_cast(ihNaN); + fNeg0 = reinterpret_cast(ihNeg0); + } + else if(is_float) + { + const uint32_t ifInf = 0x7F800000; + const uint32_t ifNegInf = 0xFF800000; + const uint32_t ifNaN = 0x7F800001; + const uint32_t ifNeg0 = 0x80000000; + fInf = reinterpret_cast(ifInf); + fNegInf = reinterpret_cast(ifNegInf); + fNaN = reinterpret_cast(ifNaN); + fNeg0 = reinterpret_cast(ifNeg0); + } + + if(x == 0) + return 0; + + uint32_t sign = x >> 7; + uint32_t mantissa = x & ((1 << wm) - 1); + int exponent = (x & 0x7F) >> wm; + if(negative_zero_nan) + { + if(x == 0x80) + return fNaN; + } + else + { + if(x == 0x80) + return fNeg0; + if(exponent == ((1 << we) - 1)) + return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN; + } + typename std::conditional::type retval; + if(we == 5 && is_half && !negative_zero_nan) + { + retval = x << 8; + return reinterpret_cast(retval); + } + + const int exp_low_cutoff + = (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0); + + //subnormal input + if(exponent == 0) + { + //guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above + int sh = 1 + clz(mantissa) - (32 - wm); + mantissa <<= sh; + exponent += 1 - sh; + /* + exponent++; + while(mantissa<(1<>= 1 - exponent; + exponent = 0; + } + + if(sizeof(T) == 2) + retval = (sign << 15) | (exponent << 10) | mantissa; + else + retval = (sign << 31) | (exponent << 23) | mantissa; + return reinterpret_cast(retval); + } + +} // namespace hip_f8_impl + +// device specific optimized code +#if defined(__gfx940__) +namespace tensile_gfx940_f8_impl +{ + template + inline HIP_DEVICE uint8_t cast_to_f8_from_f32(float v, uint32_t rng = 0) + { + uint8_t i8data; + + union + { + float fval; + uint32_t i32val; + uint8_t i8val[4]; // not endian independent + } val; + + uint32_t ival = 0; + val.fval = v; + + if(isE2M5) + { +#ifdef DOWNCAST_CLIPPING_ON + if((val.i32val & 0x7F800000) != 0x7F800000) // all exp bits are 1 --> NaN or INF + val.fval = __builtin_amdgcn_fmed3f(val.fval, 57344.0, -57344.0); +#endif + + if(stochastic_rounding) + { + ival = __builtin_amdgcn_cvt_sr_bf8_f32(val.fval, rng, ival, 0); // 0 pos + val.i32val = ival; + i8data = val.i8val[0]; // little endian + } + else // RNE CVT + { + ival = __builtin_amdgcn_cvt_pk_bf8_f32( + val.fval, val.fval, ival, false); // false -> WORD0 + val.i32val = ival; + i8data = val.i8val[0]; + } + } + else // fp8 + { +#ifdef DOWNCAST_CLIPPING_ON + if((val.i32val & 0x7F800000) != 0x7F800000) // all exp bits are 1 --> NaN or INF + val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0); +#endif + + if(stochastic_rounding) + { + ival = __builtin_amdgcn_cvt_sr_fp8_f32(val.fval, rng, ival, 0); // 0 pos + val.i32val = ival; + i8data = val.i8val[0]; // little endian + } + else // RNE CVT + { + ival = __builtin_amdgcn_cvt_pk_fp8_f32( + val.fval, val.fval, ival, false); // false -> WORD0 + val.i32val = ival; + i8data = val.i8val[0]; + } + } + return i8data; + } + +} +#endif + +// Naming convension of datatype in hip header file +// float8: fp8 +// bfloat8: bf8 +// f8 is used to consider both float8 and bfloat8 + +//namespace TensileLite +//{ +enum class hip_f8_type +{ + bf8_fnuz = 0, // 1:5:2 + fp8_fnuz = 1, // 1:4:3 + bf8 = 2, // Placeholder, not used + fp8 = 3 // Placeholder, not used +}; + +enum class hip_f8_rounding_mode +{ + standard, + stochastic +}; + +// bias mode bit implementation +// +// "bias mode optimial" +// => "bias mode bit" = 1 +// => bias = 16 for 152, 8 for 143 +// => NAN/INF are represented as negative_zero +// +// "bias mode ieee" +// => "bias mode bit" = 0 +// => bias = 15 for 152, 7 for 143 +// => NAN/INF are represented as per IEEE conventions + +// NOTE: made optimal bias mode default assuming that's the case on device +static __device__ bool hip_f8_bias_mode_bit_device = true; +static bool hip_f8_bias_mode_bit_host = true; + +static __global__ void set_hip_f8_bias_mode_bit(bool v) +{ + hip_f8_bias_mode_bit_device = v; +} + +static void set_hip_f8_bias_mode_ieee() +{ + hipLaunchKernelGGL(set_hip_f8_bias_mode_bit, dim3(1), dim3(1), 0, 0, false); + hip_f8_bias_mode_bit_host = false; +} + +static void set_hip_f8_bias_mode_optimal() +{ + hipLaunchKernelGGL(set_hip_f8_bias_mode_bit, dim3(1), dim3(1), 0, 0, true); + hip_f8_bias_mode_bit_host = true; +} + +static inline HIP_HOST_DEVICE bool get_hip_f8_bias_mode() +{ +#if defined(__HIP_DEVICE_COMPILE__) + return hip_f8_bias_mode_bit_device; +#else + return hip_f8_bias_mode_bit_host; +#endif +} + +// data type +template +struct Float8_BFloat8 +{ + uint8_t __x; + + // default constructor + HIP_HOST_DEVICE Float8_BFloat8() = default; + +#if defined(__gfx940__) + // NOTE: ON-DEVICE... always optimal bias + explicit HIP_DEVICE Float8_BFloat8(float v, + hip_f8_rounding_mode rm = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + { + // runtime branch, use default constructor and explicit_cast() if want to avoid it + + if(rm == hip_f8_rounding_mode::stochastic) + __x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(v, rng); + else + __x + = tensile_gfx940_f8_impl::cast_to_f8_from_f32(v, rng); + } + // only host code is simulated + explicit HIP_HOST +#else // gfx940 + explicit HIP_HOST_DEVICE +#endif // gfx940 + Float8_BFloat8(float v, + hip_f8_rounding_mode rm = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + { + // NOTE: made clipping default again + if(T == hip_f8_type::bf8_fnuz) + { + if(get_hip_f8_bias_mode()) + { + __x = tensile_hip_f8_impl:: + cast_to_f8<2, 5, float, true /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + else + { + __x = tensile_hip_f8_impl:: + cast_to_f8<2, 5, float, false /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + } + else /* fp8*/ + { + if(get_hip_f8_bias_mode()) + { + __x = tensile_hip_f8_impl:: + cast_to_f8<3, 4, float, true /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + else + { + __x = tensile_hip_f8_impl:: + cast_to_f8<3, 4, float, false /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + } + } + + // constructor from half + // no h/w inst for cvt from f16, just convert f16 to f32 and call constructor + explicit HIP_HOST_DEVICE Float8_BFloat8(_Float16 v, + hip_f8_rounding_mode rm + = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + : Float8_BFloat8((float)v, rm, rng) + { + } + + explicit HIP_HOST_DEVICE Float8_BFloat8(int v, + hip_f8_rounding_mode rm + = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + : Float8_BFloat8((float)v, rm, rng) + { + } + explicit HIP_HOST_DEVICE Float8_BFloat8(size_t v, + hip_f8_rounding_mode rm + = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + : Float8_BFloat8((float)v, rm, rng) + { + } + + // constructor from hip_bfloat16 + // explicit HIP_HOST_DEVICE Float8_BFloat8(hip_bfloat16 v, hip_f8_rounding_mode r=hip_f8_rounding_mode::standard, uint32_t rng=0); + + // convert to float +#if defined(__gfx940__) + // builtin conversion + explicit inline HIP_DEVICE operator float() const + { + float fval; + uint32_t i32val = static_cast(__x); + if(T == hip_f8_type::bf8_fnuz) + // workaround: use inline asm instead of builtin function + // fval = __builtin_amdgcn_cvt_f32_bf8(i32val, 0); + asm volatile("v_cvt_f32_bf8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val)); + else + // workaround: use inline asm instead of builtin function + // fval = __builtin_amdgcn_cvt_f32_fp8(i32val, 0); + asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val)); + return fval; + } + explicit inline HIP_HOST operator float() const +#else // non gfx940 + + explicit inline HIP_HOST_DEVICE operator float() const +#endif + { + if(T == hip_f8_type::bf8_fnuz) + { + if(get_hip_f8_bias_mode()) + { + return tensile_hip_f8_impl:: + cast_from_f8<2, 5, float, true /*negative_zero_nan*/>(__x); + } + else + { + return tensile_hip_f8_impl:: + cast_from_f8<2, 5, float, false /*negative_zero_nan*/>(__x); + } + } + else /* fp8*/ + { + if(get_hip_f8_bias_mode()) + { + return tensile_hip_f8_impl:: + cast_from_f8<3, 4, float, true /*negative_zero_nan*/>(__x); + } + else + { + return tensile_hip_f8_impl:: + cast_from_f8<3, 4, float, false /*negative_zero_nan*/>(__x); + } + } + } + + // convert to half + explicit inline HIP_HOST_DEVICE operator _Float16() const + { + return _Float16(float(*this)); // convert to float, then convert to f16 + } + + // convert to hip_bfloat16 + // NOTE: no hardware instruction to convert from and to f8, may want to convert it f32 first + // explicit inline HIP_HOST_DEVICE operator hip_bfloat16() const; + + // check for zero + inline HIP_HOST_DEVICE bool is_zero() const + { + if(get_hip_f8_bias_mode()) + { + return __x == 0x00; + } + else + { + return (__x == 0x00) || (__x == 0x80); + } + } + + // check for nan + inline HIP_HOST_DEVICE bool is_nan() const + { + if(get_hip_f8_bias_mode()) + { + return __x == 0x80; + } + else + { + if(T == hip_f8_type::bf8_fnuz) + { + return (__x == 0x7d) || (__x == 0x7e) || (__x == 0x7f) || (__x == 0xfd) + || (__x == 0xfe) || (__x == 0xff); + } + else + { + return (__x == 0x79) || (__x == 0x7a) || (__x == 0x7b) || (__x == 0x7c) + || (__x == 0x7d) || (__x == 0x7e) || (__x == 0x7f) || (__x == 0xf9) + || (__x == 0xfa) || (__x == 0xfb) || (__x == 0xfc) || (__x == 0xfd) + || (__x == 0xfe) || (__x == 0xff); + } + } + } + + // check for inf + inline HIP_HOST_DEVICE bool is_inf() const + { + if(get_hip_f8_bias_mode()) + { + return __x == 0x80; + } + else + { + if(T == hip_f8_type::bf8_fnuz) + { + return (__x == 0x7c) || (__x == 0xfc); + } + else + { + return (__x == 0x78) || (__x == 0xf8); + } + } + } + // + // assignment operator overloading + // + // TODO: need to verify whether it produces extra copy, need to investigate the assembly? + // use cast_to_f8 function otherwise + inline HIP_HOST_DEVICE Float8_BFloat8& operator=(const float& a) + { + __x = Float8_BFloat8(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE Float8_BFloat8& operator=(const double& a) + { + __x = Float8_BFloat8((float)a).__x; + return *this; + } + + inline __host__ __device__ Float8_BFloat8& operator=(const Float8_BFloat8& a) + { + __x = a.__x; + return *this; + } + + //inline __host__ __device__ Float8_BFloat8& operator=(const rocblas_half& a) + inline __host__ __device__ Float8_BFloat8& operator=(const _Float16& a) + { + __x = Float8_BFloat8(a).__x; + return *this; + } + + // += operator + inline __host__ __device__ Float8_BFloat8& operator+=(const Float8_BFloat8& a) + { + __x = Float8_BFloat8(float(this->__x) + float(a.__x)).__x; + return *this; + } +}; + +// TODO: place it in appropriate header +typedef Float8_BFloat8 tensile_float8_fnuz; +typedef Float8_BFloat8 tensile_bfloat8_fnuz; + +// Dummy data type for ocp (added for backward compatible header), will not be used +// Actual OCP data type will be used from new header. +typedef Float8_BFloat8 tensile_float8; +typedef Float8_BFloat8 tensile_bfloat8; + +// Other operator overloading +inline std::ostream& operator<<(std::ostream& os, const tensile_float8& f8) +{ + os << static_cast(f8); + return os; +} +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8& bf8) +{ + os << static_cast(bf8); + return os; +} +inline std::ostream& operator<<(std::ostream& os, const tensile_float8_fnuz& f8) +{ + os << static_cast(f8); + return os; +} +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8_fnuz& bf8) +{ + os << static_cast(bf8); + return os; +} + +// +inline tensile_float8 operator+(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8 operator+(tensile_float8 a, float b) +{ + return static_cast(static_cast(a) + b); +} +inline tensile_float8 operator+(float a, tensile_float8 b) +{ + return static_cast(a + static_cast(b)); +} +inline tensile_bfloat8 operator+(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8 operator-(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +inline tensile_bfloat8 operator-(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +// NOTE: It is not used in reference solution directly, we want to return float otherwise +inline tensile_float8 operator*(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} +inline tensile_float8 operator*(float a, tensile_float8 b) +{ + return static_cast(a * static_cast(b)); +} +inline tensile_float8 operator*(tensile_float8 a, float b) +{ + return static_cast(static_cast(a) * b); +} +inline tensile_bfloat8 operator*(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} + +inline tensile_float8 operator/(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline tensile_bfloat8 operator/(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline bool operator<(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<(float a, tensile_float8 b) +{ + return a < static_cast(b); +} +inline bool operator<(tensile_float8 a, float b) +{ + return static_cast(a) < b; +} +inline bool operator<(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<=(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator<=(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator==(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator==(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator!=(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator!=(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator>(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>(float a, tensile_float8 b) +{ + return a > static_cast(b); +} +inline bool operator>(tensile_float8 a, float b) +{ + return static_cast(a) > b; +} +inline bool operator>(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>=(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) >= static_cast(b); +} +inline bool operator>=(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) >= static_cast(b); +} + +// FNUZ +inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, float b) +{ + return static_cast(static_cast(a) + b); +} +inline tensile_float8_fnuz operator+(float a, tensile_float8_fnuz b) +{ + return static_cast(a + static_cast(b)); +} +inline tensile_bfloat8_fnuz operator+(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8_fnuz operator-(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +inline tensile_bfloat8_fnuz operator-(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +// NOTE: It is not used in reference solution directly, we want to return float otherwise +inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} +inline tensile_float8_fnuz operator*(float a, tensile_float8_fnuz b) +{ + return static_cast(a * static_cast(b)); +} +inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, float b) +{ + return static_cast(static_cast(a) * b); +} +inline tensile_bfloat8_fnuz operator*(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} + +inline tensile_float8_fnuz operator/(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline tensile_bfloat8_fnuz operator/(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline bool operator<(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<(float a, tensile_float8_fnuz b) +{ + return a < static_cast(b); +} +inline bool operator<(tensile_float8_fnuz a, float b) +{ + return static_cast(a) < b; +} +inline bool operator<(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<=(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator<=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator==(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator==(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator!=(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator!=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator>(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>(float a, tensile_float8_fnuz b) +{ + return a > static_cast(b); +} +inline bool operator>(tensile_float8_fnuz a, float b) +{ + return static_cast(a) > b; +} +inline bool operator>(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>=(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) >= static_cast(b); +} +inline bool operator>=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) >= static_cast(b); +} + +// ================ Explicit downcasting to support Stochastic Rounding and clipping =============== + +#if 0 // enable_if_t supported from C++14 and above, not C++11! enable it when compiler updated + template {}, int> = 0> + inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng = 0, bool clip = true) + { + // same type, no conversion + return a; + } + + // Use h/w intrinsic and optimized version when __gfx940__ + template {}), int> = 0> + inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng = 0, bool clip = true) + { +#ifdef __gfx940__ + T val; + if(std::is_same::value) + val.__x + = tensile_gfx940_f8_impl::cast_to_f8_from_f32(float(a), rng, clip); + else + val.__x + = tensile_gfx940_f8_impl::cast_to_f8_from_f32(float(a), rng, clip); + return val; +#else + return T(float(a), + stochastic_rounding ? hip_f8_rounding_mode::stochastic : hip_f8_rounding_mode::standard, + rng, + clip); +#endif + } +#else // without enable_if_t, we have to use explicit template specialization + +template +inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng = 0); + +template +inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng) +{ + // same type, no conversion + return a; +} + +// NOTE: using explicit specialization +template <> +inline __host__ __device__ tensile_float8 + explicit_downcast(float a, uint32_t rng) +{ + // Use h/w intrinsic and optimized version when __gfx940__ +#ifdef __gfx940__ + tensile_float8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_float8(float(a), hip_f8_rounding_mode::stochastic, rng); +#endif +} + +template <> +inline __host__ __device__ tensile_float8 + explicit_downcast(float a, uint32_t rng) +{ +#ifdef __gfx940__ + tensile_float8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_float8(float(a), hip_f8_rounding_mode::standard, rng); +#endif +} + +template <> +inline __host__ __device__ tensile_bfloat8 + explicit_downcast(float a, uint32_t rng) +{ +#ifdef __gfx940__ + tensile_bfloat8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_bfloat8(float(a), hip_f8_rounding_mode::stochastic, rng); +#endif +} + +template <> +inline __host__ __device__ tensile_bfloat8 + explicit_downcast(float a, uint32_t rng) +{ +#ifdef __gfx940__ + tensile_bfloat8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_bfloat8(float(a), hip_f8_rounding_mode::standard, rng); +#endif +} + +#endif // end of explicit specialization + +//} // end of namespace TensileLite + +namespace std +{ + inline bool isinf(const tensile_float8& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8& a) + { + return (static_cast(a.__x) & 0x7f) == 0x7c; + } + + inline bool isnan(const tensile_float8& a) + { + return (static_cast(a.__x) & 0x7f) == 0x7f; + } + inline bool isnan(const tensile_bfloat8& a) + { + return (static_cast(a.__x) & 0x7f) > 0x7c; + } + inline bool iszero(const tensile_float8& a) + { + return (static_cast(a.__x) & 0x7F) == 0x0; + } + inline bool iszero(const tensile_bfloat8& a) + { + return (static_cast(a.__x) & 0x7F) == 0x0; + } + + //TODO: better to & 0x7F + inline tensile_float8 abs(const tensile_float8& a) + { + return tensile_float8(std::abs(float(a))); + } + inline tensile_bfloat8 abs(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::abs(float(a))); + } + + inline tensile_float8 sin(const tensile_float8& a) + { + return tensile_float8(std::sin(float(a))); + } + inline tensile_bfloat8 sin(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::sin(float(a))); + } + + inline tensile_float8 cos(const tensile_float8& a) + { + return tensile_float8(std::cos(float(a))); + } + inline tensile_bfloat8 cos(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8& a) + { + return std::to_string(static_cast(a)); + } + // FNUZ + inline bool isinf(const tensile_float8_fnuz& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8_fnuz& a) + { + return false; + } + + inline bool isnan(const tensile_float8_fnuz& a) + { + return static_cast(a.__x) == 0x80; + } + inline bool isnan(const tensile_bfloat8_fnuz& a) + { + return static_cast(a.__x) == 0x80; + } + inline bool iszero(const tensile_float8_fnuz& a) + { + return static_cast(a.__x) == 0x0; // NOTE: only +0 exists + } + inline bool iszero(const tensile_bfloat8_fnuz& a) + { + return static_cast(a.__x) == 0x0; // NOTE: only +0 exists + } + + inline tensile_float8_fnuz abs(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::abs(float(a))); + } + inline tensile_bfloat8_fnuz abs(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::abs(float(a))); + } + + inline tensile_float8_fnuz sin(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::sin(float(a))); + } + inline tensile_bfloat8_fnuz sin(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::sin(float(a))); + } + + inline tensile_float8_fnuz cos(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::cos(float(a))); + } + inline tensile_bfloat8_fnuz cos(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + +} // namespace std diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv new file mode 100644 index 0000000000..e4c2926cbc --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv @@ -0,0 +1,2 @@ +GFlops, SizeI, SizeJ, SizeK, SizeL, LDD, LDC, LDA, LDB, TotalFlops, Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 +0, 16, 16, 1, 32, 16, 16, 16, 16, 16384, -1 diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml new file mode 100644 index 0000000000..f12bac9074 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml @@ -0,0 +1,304 @@ +- MinimumRequiredVersion: 4.33.0 +- ProblemSizes: + - Exact: [16, 16, 1, 32, 16, 16, 16, 16] +- BiasTypeArgs: [[]] +- ActivationArgs: +- 1LDSBuffer: 0 + ActivationAlt: false + ActivationFuncCall: false + ActivationFused: true + AssertAIGreaterThanEqual: -1 + AssertAILessThanEqual: -1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: true + AssignedProblemIndependentDerivedParameters: true + BaseName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x1A9fya6zl43sH9vlrOEkFz4b8sLxnJYl7JpJ1ddfe8a4= + BufferLoad: true + BufferStore: true + CUCount: null + CUOccupancy: 5 + ClusterLocalRead: 0 + CodeObjectVersion: '4' + ConvertAfterDS: false + CustomKernelName: '' + DebugStreamK: 0 + DepthU: 32 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DirectToVgprA: false + DirectToVgprB: false + DirectToVgprSparseMetadata: false + EdgeType: ShiftPtr + EnableF32XEmulationLds: false + EnableF32XdlMathOp: true + EnableMatrixInstruction: true + ExpandPointerSwap: true + ExpertSchedulingMode: 0 + ForceDisableShadowInit: false + GlobalReadPerMfma: 1 + GlobalReadVectorWidthA: 8 + GlobalReadVectorWidthB: 4 + GlobalSplitU: 1 + GlobalSplitUAlgorithm: MultipleBuffer + GlobalSplitUCoalesced: false + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 1 + GroupLoadStore: false + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + GuaranteeNoPartialMetadata: true + ISA: [9, 5, 0] + InnerUnroll: 1 + InterleaveAlpha: 0 + InternalSupportParams: {KernArgsVersion: 2, SupportCustomStaggerU: true, SupportCustomWGM: true, + SupportUserGSU: true, UseUniversalArgs: true} + Kernel: true + KernelLanguage: Assembly + KernelNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1 + LDSTrInst: false + LSCA: 16 + LSCB: 16 + LSPA: 32 + LSPB: 16 + LVCA: 2 + LVCB: 4 + LVPA: 4 + LVPB: 4 + LdsBlockSizePerPadA: 512 + LdsBlockSizePerPadB: 256 + LdsBlockSizePerPadMetadata: 0 + LdsBytesNoAmax: 13056 + LdsInitCVgprs: false + LdsNumBytes: 13056 + LdsNumElementsAlignedA: 2304 + LdsNumElementsAlignedB: 2560 + LdsNumElementsAlignedMetadata: 0 + LdsOffsetA: 0 + LdsOffsetA_Blk: 8192 + LdsOffsetB: 2304 + LdsOffsetB_Blk: 10496 + LdsOffsetBias: 0 + LdsOffsetBiasGSU: 0 + LdsOffsetBiasNonGSU: 0 + LdsOffsetMetadata: 2304 + LdsOffsetMetadata_Blk: 10496 + LdsPadA: 16 + LdsPadB: 16 + LdsPadMetadata: 0 + LocalReadVectorWidth: 4 + LocalSplitU: 1 + LocalSplitUReuseLDS: 1 + LocalWritePerMfma: -1 + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopIters: 1 + LoopUnroll: 32 + MFMA_BF16_1K: false + MIArchVgpr: false + MIBlock: [16, 16, 32, 1, 1, 1] + MIInputPerThread: 8 + MIInputPerThreadA: 8 + MIInputPerThreadB: 8 + MIInputPerThreadMetadata: 8 + MIOutputVectorWidth: 4 + MIRegPerOut: 1 + MIWaveGroup: [1, 1] + MIWaveTile: [1, 1] + MIWaveTileA: 1 + MIWaveTileB: 1 + MIWaveTileMetadata: 0 + MacroTile0: 16 + MacroTile1: 16 + MacroTileA: 16 + MacroTileB: 16 + MagicDivAlg: 2 + MathClocksUnrolledLoop: 109 + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 32 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 32, 1] + MaxOccupancy: 40 + MbskPrefetchOpt: 0 + NoLdsWriteCode: false + NoReject: false + NoTailLoop: false + NonTemporal: -1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NonTemporalD: 0 + NonTemporalE: 0 + NonTemporalMetadata: 0 + NonTemporalWS: 0 + NumElementsPerBatchStore: 0 + NumElementsPerThread: 4 + NumGlobalWriteVectorsPerThread: 4 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 64 + NumWaveSplitK: 1 + OptNoLoadLoop: 1 + PackedC0IdxChars: [I] + PackedC0IndicesX: [0] + PackedC1IdxChars: [J] + PackedC1IndicesX: [1] + PrefetchGlobalRead: 1 + PrefetchLocalRead: 0 + PreloadKernArgs: true + ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: [] + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 0 + DataTypeA: 0 + DataTypeAmaxD: 0 + DataTypeB: 0 + DataTypeE: 0 + DestDataType: 0 + F32XdlMathOp: 10 + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexAssignmentsMetadata: [3, 0, 2] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndexUnrollM: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: 0 + TransposeB: 1 + UseBeta: true + UseBias: 0 + UseE: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 3 + ScheduleLocalWrite: 1 + SolutionIndex: 0 + SolutionNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 + SourceSwap: 1 + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + StorePriorityOpt: false + StoreRemapVectorWidth: 0 + StoreSwapAddr: false + StoreSyncOpt: 0 + StoreVectorWidth: 1 + StreamK: 0 + StreamKAtomic: 0 + StreamKXCCMapping: 0 + SubGroup0: 4 + SubGroup1: 16 + SubGroupA: 4 + SubGroupB: 16 + SuppressNoLoadLoop: false + ThreadTile: [1, 1] + ThreadTile0: 4 + ThreadTile1: 1 + ThreadTileA: 4 + ThreadTileB: 1 + TransposeLDS: 0 + TransposeLDSMetadata: true + ULSGRODoubleG2L: 0 + UnrollLoopSwapGlobalReadOrder: 0 + UnrollMajorLDSA: 0 + UnrollMajorLDSB: 0 + UnrollMajorLDSMetadata: true + Use64bShadowLimit: 1 + UseDotInstruction: false + UseF32XEmulation: true + UseInstOffsetForGRO: 0 + UseSgprForGRO: -1 + Valid: true + VectorStore: -1 + VectorWidthA: 1 + VectorWidthB: 1 + WaveSeparateGlobalReadA: 0 + WaveSeparateGlobalReadB: 0 + WaveSeparateGlobalReadMetadata: 0 + WaveSplitK: false + WavefrontSize: 64 + WorkGroup: [16, 4, 1] + WorkGroupMapping: 8 + WorkGroupMappingXCC: 1 + WorkGroupMappingXCCGroup: -1 + WorkGroupReduction: false + WorkspaceCheck: [4, 0, 1] + _DepthU: 32 + _DepthUA: 32 + _DepthUB: 32 + _DepthUMetadata: 32 + _GlobalAccumulation: MultipleBuffer + _UseSgprForGRO: false + _VectorStore: 1 + _WorkspaceSizePerElemBias: 0 + _WorkspaceSizePerElemC: 4 + _staggerStrideShift: 1 + enableLDSTrA: false + enableLDSTrB: false + reorderGRInstForDTVA: false + reorderGRInstForDTVB: false + tailLoopOptA: false + tailLoopOptB: false diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index e82a07349f..dfc14a2f8c 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -111,7 +111,7 @@ def makeValidMFMA(): [16, 16, 16, 1], [4, 4, 4, 16], ] + [[32, 32, 16, 1], [16, 16, 32, 1],] + [[16, 16, 64, 1],[32, 32, 32, 1]] - validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1], [16, 16, 16, 1]] + validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1], [16, 16, 16, 1], [16, 16, 32, 1]] validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1], [32, 32, 64, 1], [16, 16, 128, 1]] validMFMA["B8"] = validMFMA["F8"] validMFMA["F8B8"] = validMFMA["F8"] diff --git a/tensilelite/Tensile/Component.py b/tensilelite/Tensile/Component.py index 5a815460e2..143d4ba3de 100644 --- a/tensilelite/Tensile/Component.py +++ b/tensilelite/Tensile/Component.py @@ -281,6 +281,12 @@ def schedIntoIteration(self, writer, kernel, tensorParametersA, tensorParameters class GlobalWriteComponents(Component): pass +class F32XEmulation(Component): + """ + Emulate F32 compute type mfma using BF16. + """ + pass + # Importing here allows auto-registry of components in the Components directory. # Each file must be listed in __all__ in Components/__init__.py # "noqa" prevents linter from complaining here. diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py new file mode 100644 index 0000000000..b63251a32f --- /dev/null +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -0,0 +1,351 @@ +################################################################################ +# +# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from rocisa import rocIsa +from rocisa.code import Module, TextBlock +from rocisa.container import vgpr, sgpr,SDWAModifiers, VOP3PModifiers +from rocisa.enum import SelectBit, UnusedBit +from rocisa.instruction import VAdd3U32, VCvtF32toF16, VLShiftRightB32, \ + VCmpUF32, VCndMaskB32, VCvtPkF32toFP8, VCvtPkF32toBF8, \ + VCmpClassF32, VOrB32, VPackF16toB32, \ + VAndOrB32, VBfeU32, VLShiftLeftB16, SNop, VMed3F32, \ + VCvtPkF32toBF16, VAndB32, \ + VMovB32, VLShiftLeftB32, VSubF32, MFMAInstruction +from ..TensileInstructions import * +from ..TensileInstructions import VCvtBF16toFP32 +#from ..TensileInstructions.Instructions import * +from ..TensileInstructions import DataType, \ + SaturateCastType, VSaturateCastInt + +from ..Component import F32XEmulation + +import re, types + +class F32XEmulationCvtLocalWrite(F32XEmulation): + asmCaps = {"HasMFMA_xf32": True} + dbgCounter = 0 + def __call__(self, srcStart): + tf32mod = Module() + tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) + if (F32XEmulationCvtLocalWrite.dbgCounter == 0): + tf32mod.add(TextBlock(str("label_tf32lds_begin_") + str(F32XEmulationCvtLocalWrite.dbgCounter) + ":\n")) + # From: + # + # 0: G2LA+0 = 0, 4, 8, 12 + # 1: G2LA+1 = 1, 5, 9, 13 + # 2: G2LA+2 = 2, 6, 10, 14 + # 3: G2LA+3 = 3, 7, 11, 15 + # + # To: + # + # 0: [0high, 0low] + # 1: [1high, 1low] + # 2: [2high, 2low] + # 3: [3high, 3low] + # + # Carson: cannot do this, as it will break the 4 stride reassembly + # 0: [0high, 1high] + # 1: [2high, 3high] + # 2: [0low, 1low] + # 3: [2low, 3low] + # + + + # # high bits + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(srcStart), src1=vgpr(srcStart + "+1"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(srcStart + "+2"), src1=vgpr(srcStart + "+3"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # low bits + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr(srcStart), src1=vgpr("Cvt+8"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr(srcStart + "+1"), src1=vgpr("Cvt+9"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr(srcStart + "+2"), src1=vgpr("Cvt+8"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr(srcStart + "+3"), src1=vgpr("Cvt+9"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+0"), src0=vgpr("Cvt+2"), src1=vgpr(srcStart + "+0"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+1"), src0=vgpr("Cvt+3"), src1=vgpr(srcStart + "+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+2"), src0=vgpr("Cvt+4"), src1=vgpr(srcStart + "+2"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+3"), src0=vgpr("Cvt+5"), src1=vgpr(srcStart + "+3"))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + + # tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VMovB32(dst=vgpr("G2LA+1"), src=vgpr("Cvt+1"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+3"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+5"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + if (F32XEmulationCvtLocalWrite.dbgCounter == 0): + tf32mod.add(TextBlock(str("label_tf32lds_end_") + str(F32XEmulationCvtLocalWrite.dbgCounter) + ":\n")) + F32XEmulationCvtLocalWrite.dbgCounter += 1 + + return tf32mod + +def issueLatencyOp(self): + return + +class F32XEmulationCvtLocalRead(F32XEmulation): + asmCaps = {"HasMFMA_xf32": True} + dbgCounter = 0 + def __call__(self, dstStart): + tf32mod = Module() + # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? + tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #tf32mod.add(SNop(waitState=27, comment="wait for ds_read")) + tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) + F32XEmulationCvtLocalRead.dbgCounter += 1 + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("Cvt+0"), src=vgpr(dstStart + "+0"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+1"), src=vgpr(dstStart + "+1"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+2"), src=vgpr(dstStart + "+2"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+3"), src=vgpr(dstStart + "+3"))) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + + #pack high bits + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #pack low bits + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + + # read format: + # 0: [0high, 1high] + # 1: [2high, 3high] + # 2: [0low, 1low] + # 3: [2low, 3low] + # + + tf32mod.add(TextBlock(str("label_tf32Read_end_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) + F32XEmulationCvtLocalRead.dbgCounter += 1 + return tf32mod + + +class F32XEmulationMFMA(F32XEmulation): + asmCaps = {"HasMFMA_xf32": True} + #kernel = {"ProblemType": {"DataType": "F32XdlMathOp"}, "UseF32XEmulation"} + dbgCounter = 0 + # width = vgprPerInput (4 or 8) + def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k, width, neg_flag): + tf32mod = Module() + tf32mod.add(TextBlock("/*tf32 emulation*/\n")) + aStart = re.search(r'v\[vgpr(.*?):', str(src1)).group(1) + bStart = re.search(r'v\[vgpr(.*?):', str(src0)).group(1) + # print("src0: {0} aStart: {1}".format(src0, aStart)) + # print("src1: {0} bStart: {1}".format(src1, bStart)) + tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) + tf32mod.add(TextBlock("/*bf16AHigh*/\n")) + numElementsPerIter = 4 + vgprSize = width / 2 + + # Cvt register layout: + #cvt0-1: Tmp + #cvt2-5: AHigh + #cvt6-9: Bhigh + #cvt12-15: ALow + #cvt16-19: BLow + + aHigh = "Cvt+2" + bHigh = "Cvt+6" + aLow = "Cvt+12" + bLow = "Cvt+16" + + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + + for itr in range(int(width / numElementsPerIter)): + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + offset = "+" + str(itr * 2) + aHigh1 = aHigh + offset + aHigh2 = aHigh + "+1" + offset + aLow1 = aLow + offset + aLow2 = aLow + "+1" + offset + bHigh1 = bHigh + offset + bHigh2 = bHigh + "+1" + offset + bLow1 = bLow + offset + bLow2 = bLow + "+1" + offset + tmp1 = "Cvt+0" + tmp2 = "Cvt+1" + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(TextBlock("/*bf16AHigh*/\n")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh1), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh2), src0=vgpr(aStart + "+2"), src1=vgpr(aStart + "+3"))) + tf32mod.add(TextBlock("/*bf16BHigh*/\n")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh1), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh2), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) + + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + if kernel["EnableF32XEmulationLds"]: + (src0, src1) = (vgpr(bStart,vgprSize), vgpr(aStart,vgprSize)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + else: + (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2)) + + for itr in range(int(width / numElementsPerIter)): + offset = "+" + str(itr * 2) + aHigh1 = aHigh + offset + aHigh2 = aHigh + "+1" + offset + aLow1 = aLow + offset + aLow2 = aLow + "+1" + offset + bHigh1 = bHigh + offset + bHigh2 = bHigh + "+1" + offset + bLow1 = bLow + offset + bLow2 = bLow + "+1" + offset + tmp1 = "Cvt+0" + tmp2 = "Cvt+1" + if kernel["EnableF32XEmulationLds"]: + None + #tf32mod.add(SNop(waitState=1020, comment="1 wait states for ds_read")) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + #(src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + #tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + + tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + if kernel["EnableF32XEmulationLds"]: + (src0, src1) = (vgpr(bStart,vgprSize), vgpr(aStart + "+" + str(vgprSize),vgprSize)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + else: + (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) + # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SNop(waitState=64, comment="1 wait s00tates for ds_read")) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + + for itr in range(int(width / numElementsPerIter)): + offset = "+" + str(itr * 2) + aHigh1 = aHigh + offset + aHigh2 = aHigh + "+1" + offset + aLow1 = aLow + offset + aLow2 = aLow + "+1" + offset + bHigh1 = bHigh + offset + bHigh2 = bHigh + "+1" + offset + bLow1 = bLow + offset + bLow2 = bLow + "+1" + offset + tmp1 = "Cvt+0" + tmp2 = "Cvt+1" + + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + + aStart = aStart + "+4" + bStart = bStart + "+4" + + #todo: working impl using in situ cvt cmd. lds currently some kernels are failing + if kernel["EnableF32XEmulationLds"]: + # tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + # (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + (src0, src1) = (vgpr(bStart + "+" + str(vgprSize),vgprSize), vgpr(aStart,vgprSize)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + else: + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) + (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + # (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) + # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2)) # tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + # tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + # (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) + # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SNop(waitState=64, comment="1 wait s00tates for ds_read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + + tf32mod.add(TextBlock(str("label_tf32_") + str(F32XEmulationMFMA.dbgCounter) + ":\n")) + F32XEmulationMFMA.dbgCounter += 1 + return tf32mod \ No newline at end of file diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index 417f8e4a7d..217809871c 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -22,13 +22,14 @@ # ################################################################################ -from rocisa.code import Module +from rocisa.code import Module, TextBlock from rocisa.container import DSModifiers, vgpr, sgpr, SDWAModifiers, VOP3PModifiers from rocisa.enum import SelectBit from rocisa.instruction import SMovB32, SWaitCnt, VOrB32, VPermB32, VLShiftLeftOrB32, \ VMovB32, VLShiftRightB32, VCvtPkFP8toF32, VCvtF32toF16, VCvtFP8toF32,VCvtScaleFP8toF16,VCvtScalePkFP8toF16 -from ..Component import LocalRead +from ..Component import LocalRead, F32XEmulation +from ..Components.F32XEmulation import F32XEmulationCvtLocalRead from math import ceil @@ -239,6 +240,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): for vIdx in range(0, numVectorsPerTile): for eIdx in range(0, numReadsPerVector): valuiIdx = int(valufIdx) + baseValuiIdx = valuiIdx localReadCode = imod.add(Module("LocalRead%s Valu%u"%(tc,valuiIdx))) if needPack or numSplitMetadata: packCode = pack.add(Module("packCode")) @@ -548,6 +550,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): if (kernel["LdsBlockSizePerPad%s"%tc] != 0) and (kernel["LdsPad%s"%tc] != 0): offset_val = offset_val + (offset_val // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpeDS"] offset_val = offset_val + tP["localReadSwapByteOffset"] + # offset_val = offset_val * 4 #Carson: Debug if (kernel["DirectToLds%s" % tc] and \ kernel["GlobalReadVectorWidth%c"%tc] * tP["bpeDS"] > 4): # another address conversion for DirectToLds + NumLoadsCoalesced > 1 @@ -621,26 +624,14 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): elif kernel["ProblemType"]["DataType"].isSingle(): localReadCode.add(writer.assert_eq( dbgVgpr, 1.0) ) + if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: + tf32mod = F32XEmulationCvtLocalRead() + dstStart = "Valu%s_X%u_I%u+%u"%(tc, bufferIdx, iui, baseValuiIdx) + localReadCode.add(tf32mod(dstStart)) + + # DTV case, do not return local read code. Return pack code only. if (tP["isA"] or tP["isB"]) and kernel["DirectToVgpr%s"%tc]: imod = Module("LocalReadDo%s_I%s (Empty)" % (tP["tensorChar"],iui)) - if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"] and tP["isA"] and tc == "A": - tf32mod = Module() - tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - localReadCode.add(tf32mod) - return imod, pack diff --git a/tensilelite/Tensile/Components/LraTileAssignment.py b/tensilelite/Tensile/Components/LraTileAssignment.py index e84a9dd70b..299d2b2585 100644 --- a/tensilelite/Tensile/Components/LraTileAssignment.py +++ b/tensilelite/Tensile/Components/LraTileAssignment.py @@ -26,7 +26,7 @@ from rocisa.container import vgpr, ContinuousRegister from rocisa.instruction import VAddU32, vectorStaticRemainder, \ vectorStaticDivideAndRemainder, vectorStaticDivide, vectorStaticMultiply, \ - vectorStaticMultiplyAdd + vectorStaticMultiplyAdd, VLShiftLeftAddU32 from ..Component import LraTileAssignment, LraTileProperties from dataclasses import dataclass @@ -287,9 +287,13 @@ def LraTileAssignmentCode(self, writer, kernel, tP, tReg, kReg, tmpVgprRes, divi comment="5.2 offset in wave: lrOffset = bnOffset + lrKOffset")) module.add(VAddU32(dst=vgpr(tReg), src0=vgpr(kReg), src1=vgpr(tReg), \ comment="6. offset in wave: lrOffset = bnOffset + lrKOffset")) + elif (kernel["MIInputPerThreadA"] == 8): + # 256b reads require additional offset, offset += int(serialId / 16) * 16 + module.add(VLShiftLeftAddU32(dst=vgpr(tReg), shiftHex=3, src0=vgpr(kReg), src1=vgpr(tReg), \ + comment="256b offset")) else: module.add(vectorStaticMultiplyAdd(vgpr(tReg), vgpr(kReg), strideK, vgpr(tReg), tmpSgprInfo, \ - "5. K offset: lrKOffset = kIdx * mStride(%u); 6. offset in wave: lrOffset = bnOffset + lrKOffset" % (strideK))) + "5. K offset: lrKOffset = kIdx * mStride(%u); 6. offset in wave: lrOffset = bnOffset + lrKOffset" % (strideK))) # wave offset if num1DWaves > 1: diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 6aa77c84dc..17d91a30db 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -1166,6 +1166,7 @@ def fixupBatch(self, writer, kernel, ss, batchIdx, edge, gwvw, \ # module.addComment1("WavefrontSize={}, WaveNum={}, storeWidth={}, bpeC={}".format(kernel["WavefrontSize"], WaveNum, storeWidth, writer.states.bpeCinternal)) module.add(SAddU32(dst=sgpr(tmpS01), src0=sgpr(tmpS01), src1=increment, comment="Inc sgpr offset")) + print("carsonStreamK") module.add(writer.readInput(kernel, ss, 'WS', kernel["ProblemType"]["ComputeDataType"], addrCalc, vc0, data, gwvw, addrCVgpr, sgpr(tmpS01))) loadsIssued += 1 diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 87b2143028..83397a1051 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1084,6 +1084,8 @@ def checkLocalReadFIFOFull(currentMFMA, fifo, lrItems, numLR, numLREven): insertedPackM = 0 def hasDependency(lr: DSLoadInstruction, inst: Instruction) -> bool: + if not (hasattr(lr, 'dst') and callable(lr.dst)): + return False lrDataReg = lr.dst if isinstance(inst, MFMAInstruction): @@ -1119,8 +1121,9 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): latencyLeft -= (tPA["localWriteInstruction"].issueLatency*2) readLeftLROPT = 0 for j in range(len(localReadItemsThisLoop)): - latencyLeft -= localReadItemsThisLoop[j].issueLatency()*2 - readLeftLROPT += 1 if latencyLeft >= 0 else 0 + if (hasattr(localReadItemsThisLoop[j], "issueLatency")): + latencyLeft -= localReadItemsThisLoop[j].issueLatency()*2 + readLeftLROPT += 1 if latencyLeft >= 0 else 0 # at least 1 instruction readLeftLROPT = max(readLeftLROPT,1) # evenly schedule localread with each mfma @@ -1160,9 +1163,11 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): latencyLeft -= readLeft * tPA["localWriteInstruction"].issueLatency * 2 else: readLeft = len(localReadItemsThisLoop) - latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) + if (hasattr(j, "issueLatency")): + latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) else: - latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) + if (hasattr(j, "issueLatency")): + latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) # force to schedule all remaining localreads before start to schedule localwrite. if mfmaIndex == self.states.sync1LdsMfmaIndex and oneBufferScheduling: @@ -1296,8 +1301,9 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): if self.states.numItersPLR and iteration >= isBarrier: readLeftLROPT = 0 for j in range(len(localReadItemsNextLoop)): - latencyLeft -= localReadItemsNextLoop[j].issueLatency()*2 - readLeftLROPT += 1 if latencyLeft >= 0 else 0 + if (hasattr(localReadItemsNextLoop[j], "issueLatency")): + latencyLeft -= localReadItemsNextLoop[j].issueLatency()*2 + readLeftLROPT += 1 if latencyLeft >= 0 else 0 # at least 1 instruction readLeftLROPT = max(readLeftLROPT,1) readLeftLREven = numReadsInst / (numMfmaPerIter - i) @@ -4399,7 +4405,7 @@ def readWriteVectors(mat, vw, kernel): #align 64 bit vgprIdx = int((vgprIdx + 1) / 2) * 2 self.states.startVgprCvt = vgprIdx - vgprIdx += 9 # for vgpr serial id + vgprIdx += 20 # for vgpr serial id self.states.totalVgprs = max(vgprIdx, self.states.c.numVgprValu) if self.states.totalVgprs < 0 or self.states.totalVgprs > self.states.regCaps["MaxVgpr"]: diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 0039c6b999..8027a6f305 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -74,6 +74,7 @@ VPrngB32, VReadfirstlaneB32, VSubF32, VSubI32, VSubU32, VXorB32 from .Component import Component +from .Components.F32XEmulation import F32XEmulationMFMA, F32XEmulationCvtLocalWrite from .KernelWriterModules import * from .SolutionStructs import isPackedIndex from .AsmStoreState import StoreState, VectorDataTypes @@ -106,6 +107,9 @@ class TailOptParams: firstLoop: int = 0 finalLoop: int = 0 +dbgCounter = 0 +dbgCounterMFMA = 0 + ################################################################################ # Assembly Kernel ################################################################################ @@ -3990,7 +3994,6 @@ def lraFinalOffset(self, kernel, tP): with self.allocTmpSgpr(1) as tmpSgprInfo: module.add(vectorStaticMultiplyAdd(vgpr("LocalReadAddr%s"%tc), vgpr(rReg), kernel["LdsPad%s"%tc] * tP["bpeDS"], vgpr("LocalReadAddr%s"%tc), tmpSgprInfo, \ "Final Offset: padding %u per block %u" % (kernel["LdsPad%s"%tc] * tP["bpeDS"], kernel["LdsBlockSizePerPad%s"%tc]))) - # release resources self.vgprPool.checkIn(tmpVgpr) self.vgprPool.checkIn(wave_id) @@ -6218,6 +6221,7 @@ def accVgprReadWriteIndex(self, kernel, idx, sz=1): else: return vgpr(idx, sz) + ############################################################################## # MFMA Iteration ############################################################################## @@ -6763,75 +6767,23 @@ def mfmaIter(self, kernel, tPA, tPB, u, innerUnroll, vregSetIdx, unrollLoopIdx = comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) else: # TF32 Emulation - if miInInstType == InstType.INST_XF32 and not self.states.asmCaps["HasMFMA_xf32"] and kernel["UseF32XEmulation"]: - tf32mod = Module() + if kernel["UseF32XEmulation"]: acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)) acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)) - tf32mod.add(TextBlock("/*tf32 emulation*/\n")) - tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) - tf32mod.add(TextBlock("/*bf16AHigh*/\n")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(aStr_base), src1=vgpr(aStr_base + "+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(aStr_base + "+2"), src1=vgpr(aStr_base + "+3"))) - tf32mod.add(TextBlock("/*bf16BHigh*/\n")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+2"), src0=vgpr(bStr_base), src1=vgpr(bStr_base + "+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+3"), src0=vgpr(bStr_base + "+2"), src1=vgpr(bStr_base + "+3"))) - tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStr_base+"+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStr_base+"+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+4"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStr_base+"+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStr_base+"+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+5"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+2", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStr_base+"+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+2", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStr_base+"+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+6"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+3", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStr_base+"+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+3", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStr_base+"+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+7"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - - #todo: working impl using in situ cvt cmd. lds currently some kernels are failing - if kernel["EnableF32XEmulationLds"]: - tf32mod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - else: - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+0",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) - (src0, src1) = (vgpr("Cvt+6",2), vgpr("Cvt+0",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+4",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - imod.add(tf32mod) + emulation = F32XEmulationMFMA() + #mfma_1k = True + imod.add(emulation(kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k,\ + vgprPerInputA, neg_flag=neg_flag)) + imod.add(TextBlock("/*mfma ops: {0} {1}*/\n".format(str(src0), str(src1)))) else: imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) prevAccIdx = accIdx + global dbgCounterMFMA + imod.add(TextBlock(str("label_mfma_") + str(dbgCounterMFMA) + ":\n")) + dbgCounterMFMA += 1 if kernel["ExpertSchedulingMode"] > 0: imod.add(SWaitAlu(va_vdst=0, comment="wait for the current iter's writes to complete")) @@ -8316,6 +8268,8 @@ def globalReadBody(tP): glc=isGlc, slc=isSlc, nt=isNT, lds=isLds, \ hi16=isHigh16Bits , \ comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp))) + if tc == 'A' and (kernel["MIInputPerThreadA"] == 8): + instOffset += 16 # 256b reads require 16B offset if unrollMirrorWithSoffset: codeMod = Module("mirrorIdx%u"%loopCnt) @@ -9278,27 +9232,6 @@ def localWriteBody(tP): else: printExit("Unsupported combination DataType%s (%s) -> DataType (%s)"%(tc, kernel["ProblemType"]["DataType%s"%tc].toChar(), kernel["ProblemType"]["DataType"].toChar())) - if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: - vgrStr = str("vgpr" + destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) - if "A" in vgrStr: - tf32mod = Module() - tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr("G2LA+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr("G2LA+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr("G2LA+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) - localWriteCode.add(tf32mod) - LocalWriteX = tP["localWriteInstruction"].getInst(isHigh16Bits) if numBlocks == 1: if (paramList[1] >= 0x20000): @@ -9316,6 +9249,11 @@ def localWriteBody(tP): else: ds = DSModifiers(na=2, offset0=paramList[2], offset1=paramList[3]) writeInst = LocalWriteX(dstAddr=vgpr(lwa), src0=paramList[0], src1=paramList[1], ds=ds, comment=comment) + if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: + vgrStr = str(destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) + # if "A" in vgrStr: + emulationLocalWrite = F32XEmulationCvtLocalWrite() + localWriteCode.add(emulationLocalWrite(vgrStr)) if self.do["LocalWriteCVT"]: localWriteCode.add(localWriteCVTCode) if self.do["LocalWrite%s"%tc]: @@ -11859,6 +11797,7 @@ def bufferLoadImpl(soffset): # split into two dwordx4 loads. Second load offset is +0.5 bpl rv = Module("emulated _buffer_load_b256") dst = None if lds else vgpr(destVgpr, rpv//2) + print("emulatedb256 ", dst, addr0) rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ soffset=soffset, mubuf=mubuf, comment=comment)) mubuf2 = MUBUFModifiers(offen=True, offset12=int(offset + bpl/2), glc=glc, slc=slc, nt=nt, lds=lds) @@ -11867,8 +11806,10 @@ def bufferLoadImpl(soffset): elif isinstance(destVgpr, int): dst2 = int(destVgpr + int(rpv//2)) dst = None if lds else vgpr(dst2, rpv//2) - rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ - soffset=soffset, mubuf=mubuf2, comment=comment)) + print("emulatedb2562 ", dst, addr0) + # rv.add(SWaitCnt(vmcnt=0, comment="")) + # rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ + # soffset=soffset, mubuf=mubuf2, comment=comment)) return rv elif bpl==64 and not lds: rv = Module("emulated _buffer_load_b512") @@ -11899,6 +11840,11 @@ def bufferLoadImpl(soffset): soffset=soffset, mubuf=mubuf4, comment=comment)) else: assert 0, "%s\nchooseGlobalRead: bad bpl %u"%(self.states.kernelName,bpl) + + # global dbgCounter + # rv.add(SWaitCnt(lgkmcnt=0, comment="")) + # rv.add(TextBlock(str("label_gr_") + str(dbgCounter) + ":\n")) + # dbgCounter += 1 # buffer_load offset field is 12-bit. # if offset >= 4096, use soffset instead diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 2c2944e2fb..f24b8d213b 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -891,7 +891,7 @@ def assignDerivedParameters( # the keys "EnableF32XdlMathOp" and "F32XdlMathOp". state["EnableF32XdlMathOp"] = False state["UseF32XEmulation"] = False #enable emulation for missing hardware support - state["EnableF32XEmulationLds"] = False + state["EnableF32XEmulationLds"] = True #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ @@ -1499,6 +1499,7 @@ def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: ldsPadB = state["LdsPadB"] optPadA = optPadB = lrvw readRegsA = readRegsB = lrvw * state["ProblemType"]["DataType"].numBytes() // 4 + print("calcLds, {0}, {1}".format(readRegsA, readRegsB)) if state["ProblemType"]["Sparse"]: if state["ProblemType"]["Sparse"] == 2: optPadB //= 2 @@ -1507,6 +1508,7 @@ def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: optPadA //= 2 readRegsA //= 2 if (not isaInfoMap[isa].asmCaps['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): + print("error1, {0}, {1}".format(readRegsA, readRegsB)) reject(state, "LocalReadVectorWidth results in attemping to read LDS larger than b128, reject") return if state["EnableMatrixInstruction"]: @@ -1586,6 +1588,7 @@ def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: if state["DirectToVgprB"]: ldsPadB = 0 + print("values: {0}, {1}, {2}".format(ldsPadA, ldsPadB, ldsPadM)) return ldsPadA, ldsPadB, ldsPadM def calcLdsBlockSizePerPad(lrvw: int) -> int: @@ -1707,9 +1710,9 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo if state["ProblemType"]["Sparse"] and state["MIInputPerThread"] * state["ProblemType"]["DataType"].numBytes() > 16: if state["LocalReadVectorWidth"] < state["MIInputPerThread"] // 2: reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"] // 2)) - elif not state["ProblemType"]["Sparse"] and not(state["ProblemType"]["DataType"].is8bitFloat() and (state["MatrixInstK"] == 64 or state["MatrixInstK"] == 128)): - if state["LocalReadVectorWidth"] < state["MIInputPerThread"]: - reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) + #elif not state["ProblemType"]["Sparse"] and not(state["ProblemType"]["DataType"].is8bitFloat() and (state["MatrixInstK"] == 64 or state["MatrixInstK"] == 128)): + #if state["LocalReadVectorWidth"] < state["MIInputPerThread"]: + # reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) if state["LocalReadVectorWidth"] > state["MIInputPerThread"] and not state["TransposeLDS"]: reject(state, printRejectionReason, "LocalReadVectorWidth require Transpose LDS") @@ -1865,8 +1868,8 @@ def calSwizzleK(state, tc): reject(state, printRejectionReason, "VWB * DataType.numBytes() > 16") # reject - GRVW too big - if (state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataTypeA"].numBytes()) > 16: - reject(state, printRejectionReason, "GRVWA * DataTypeA.numBytes() > 16") + #if (state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataTypeA"].numBytes()) > 16: + # reject(state, printRejectionReason, "GRVWA * DataTypeA.numBytes() > 16") if (state["GlobalReadVectorWidthB"] * state["ProblemType"]["DataTypeB"].numBytes()) > 16: reject(state, printRejectionReason, "GRVWB * DataTypeB.numBytes() > 16") @@ -2016,16 +2019,8 @@ def calSwizzleK(state, tc): reject(state, printRejectionReason, "Not implement DTVSM with VW>1") break - # f32 emulation currently only supports a limited set of solutions if state["UseF32XEmulation"]: - if isaInfoMap[isa].archCaps["HasF32XEmulation"]: - if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1 : - reject(state, "Missing implementation for F32X Emulation VW>1") - break - if depthU != 16: - reject(state, "Missing implementation for F32X Emulation DepthU!=16") - break - else: + if not isaInfoMap[isa].archCaps["HasF32XEmulation"]: reject(state, "Missing emulation for F32X") break