From adc868bc7ce28fbcd2870cfb05891f8d0894dfbc Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Tue, 15 Apr 2025 16:46:23 -0500 Subject: [PATCH 01/18] adding f32xemulation component --- clients/gtest/matmul_gtest.yaml | 4 +- tensilelite/Tensile/Component.py | 6 + .../Tensile/Components/F32XEmulation.py | 147 ++++++++++++++++++ tensilelite/Tensile/Components/LocalRead.py | 21 +-- tensilelite/Tensile/KernelWriterAssembly.py | 86 +--------- .../Tensile/SolutionStructs/Solution.py | 2 + 6 files changed, 169 insertions(+), 97 deletions(-) create mode 100644 tensilelite/Tensile/Components/F32XEmulation.py diff --git a/clients/gtest/matmul_gtest.yaml b/clients/gtest/matmul_gtest.yaml index 71e88fc94b..ee611e2325 100644 --- a/clients/gtest/matmul_gtest.yaml +++ b/clients/gtest/matmul_gtest.yaml @@ -1247,7 +1247,7 @@ Tests: beta: [ 0.0, 2.0 ] unit_check: 1 algo_method: [2] - gpu_arch: '942' + gpu_arch: '9(42 | 50)' - name: matmul_groupedgemm_zero_n category: pre_checkin @@ -1648,7 +1648,7 @@ Tests: alpha: 1 beta: [ 0.0, 2.0 ] unit_check: 1 - gpu_arch: '942' + gpu_arch: '9(42 | 50)' - name: matmul_gemm_double category: pre_checkin diff --git a/tensilelite/Tensile/Component.py b/tensilelite/Tensile/Component.py index 5a815460e2..143d4ba3de 100644 --- a/tensilelite/Tensile/Component.py +++ b/tensilelite/Tensile/Component.py @@ -281,6 +281,12 @@ def schedIntoIteration(self, writer, kernel, tensorParametersA, tensorParameters class GlobalWriteComponents(Component): pass +class F32XEmulation(Component): + """ + Emulate F32 compute type mfma using BF16. + """ + pass + # Importing here allows auto-registry of components in the Components directory. # Each file must be listed in __all__ in Components/__init__.py # "noqa" prevents linter from complaining here. diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py new file mode 100644 index 0000000000..2aa0073a86 --- /dev/null +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -0,0 +1,147 @@ +################################################################################ +# +# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from rocisa import rocIsa +from rocisa.code import Module, TextBlock +from rocisa.container import vgpr, sgpr,SDWAModifiers, VOP3PModifiers +from rocisa.enum import SelectBit, UnusedBit +from rocisa.instruction import VAdd3U32, VCvtF32toF16, VLShiftRightB32, \ + VCmpUF32, VCndMaskB32, VCvtPkF32toFP8, VCvtPkF32toBF8, \ + VCmpClassF32, VOrB32, VPackF16toB32, \ + VAndOrB32, VBfeU32, VLShiftLeftB16, SNop, VMed3F32, \ + VCvtPkF32toBF16, VAndB32, \ + VMovB32, VLShiftLeftB32 +from ..TensileInstructions import * +from ..TensileInstructions import VCvtBF16toFP32 +from ..TensileInstructions.Instructions import * +from ..TensileInstructions import DataType, \ + SaturateCastType, VSaturateCastInt + +from ..Component import F32XEmulation + +import re + +class F32XEmulationCvtLocalWrite(F32XEmulation): + asmCaps = {"HasMFMA_xf32": True} + def __call__(self): + tf32mod = Module() + tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr("G2LA+0"), src1=vgpr("Cvt+8"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr("G2LA+1"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr("G2LA+2"), src1=vgpr("Cvt+8"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + return tf32mod + +class F32XEmulationCvtLocalRead(F32XEmulation): + asmCaps = {"HasMFMA_xf32": True} + def __call__(self): + tf32mod = Module() + tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + return tf32mod + + +class F32XEmulationMFMA(F32XEmulation): + asmCaps = {"HasMFMA_xf32": True} + #kernel = {"ProblemType": {"DataType": "F32XdlMathOp"}, "UseF32XEmulation"} + dbgCounter = 0 + def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k, neg_flag): + tf32mod = Module() + tf32mod.add(TextBlock("/*tf32 emulation*/\n")) + aStart = re.search(r'v\[vgpr(.*?):', str(src1)).group(1) + bStart = re.search(r'v\[vgpr(.*?):', str(src0)).group(1) + tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) + tf32mod.add(TextBlock("/*bf16AHigh*/\n")) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(aStart + "+2"), src1=vgpr(aStart + "+3"))) + tf32mod.add(TextBlock("/*bf16BHigh*/\n")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+2"), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+3"), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) + tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStart+"+0"), src1=vgpr("Cvt+8"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStart+"+1"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+4"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStart+"+2"), src1=vgpr("Cvt+8"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStart+"+3"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+5"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) + tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+2", vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStart+"+0"), src1=vgpr("Cvt+8"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+2", vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStart+"+1"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+6"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+3", vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStart+"+2"), src1=vgpr("Cvt+8"))) + tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+3", vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStart+"+3"), src1=vgpr("Cvt+9"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+7"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) + + #todo: working impl using in situ cvt cmd. lds currently some kernels are failing + if kernel["EnableF32XEmulationLds"]: + tf32mod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2)) + else: + tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+4",2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) + (src0, src1) = (vgpr("Cvt+6",2), vgpr("Cvt+0",2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+0",2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2)) + + tf32mod.add(TextBlock(str("label_tf32_") + str(F32XEmulationMFMA.dbgCounter) + ":\n")) + F32XEmulationMFMA.dbgCounter += 1 + return tf32mod \ No newline at end of file diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index 417f8e4a7d..07f96e2bb9 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -28,7 +28,8 @@ from rocisa.instruction import SMovB32, SWaitCnt, VOrB32, VPermB32, VLShiftLeftOrB32, \ VMovB32, VLShiftRightB32, VCvtPkFP8toF32, VCvtF32toF16, VCvtFP8toF32,VCvtScaleFP8toF16,VCvtScalePkFP8toF16 -from ..Component import LocalRead +from ..Component import LocalRead, F32XEmulation +from ..Components.F32XEmulation import F32XEmulationCvtLocalRead from math import ceil @@ -626,21 +627,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): imod = Module("LocalReadDo%s_I%s (Empty)" % (tP["tensorChar"],iui)) if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"] and tP["isA"] and tc == "A": - tf32mod = Module() - tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - localReadCode.add(tf32mod) + tf32mod = F32XEmulationCvtLocalRead() + localReadCode.add(tf32mod()) return imod, pack diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 0039c6b999..85879b3bb3 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -74,6 +74,7 @@ VPrngB32, VReadfirstlaneB32, VSubF32, VSubI32, VSubU32, VXorB32 from .Component import Component +from .Components.F32XEmulation import F32XEmulationMFMA, F32XEmulationCvtLocalWrite from .KernelWriterModules import * from .SolutionStructs import isPackedIndex from .AsmStoreState import StoreState, VectorDataTypes @@ -6218,6 +6219,7 @@ def accVgprReadWriteIndex(self, kernel, idx, sz=1): else: return vgpr(idx, sz) + ############################################################################## # MFMA Iteration ############################################################################## @@ -6763,69 +6765,12 @@ def mfmaIter(self, kernel, tPA, tPB, u, innerUnroll, vregSetIdx, unrollLoopIdx = comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) else: # TF32 Emulation - if miInInstType == InstType.INST_XF32 and not self.states.asmCaps["HasMFMA_xf32"] and kernel["UseF32XEmulation"]: - tf32mod = Module() + if kernel["UseF32XEmulation"]: acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)) acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)) - tf32mod.add(TextBlock("/*tf32 emulation*/\n")) - tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) - tf32mod.add(TextBlock("/*bf16AHigh*/\n")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(aStr_base), src1=vgpr(aStr_base + "+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(aStr_base + "+2"), src1=vgpr(aStr_base + "+3"))) - tf32mod.add(TextBlock("/*bf16BHigh*/\n")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+2"), src0=vgpr(bStr_base), src1=vgpr(bStr_base + "+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+3"), src0=vgpr(bStr_base + "+2"), src1=vgpr(bStr_base + "+3"))) - tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStr_base+"+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStr_base+"+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+4"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStr_base+"+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStr_base+"+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+5"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+2", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStr_base+"+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+2", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStr_base+"+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+6"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+3", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStr_base+"+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+3", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStr_base+"+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+7"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - - #todo: working impl using in situ cvt cmd. lds currently some kernels are failing - if kernel["EnableF32XEmulationLds"]: - tf32mod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - else: - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+0",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) - (src0, src1) = (vgpr("Cvt+6",2), vgpr("Cvt+0",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+4",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ - a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ - comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) - imod.add(tf32mod) + emulation = F32XEmulationMFMA() + imod.add(emulation(kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k,\ + neg_flag=neg_flag)) else: imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ @@ -9281,23 +9226,8 @@ def localWriteBody(tP): if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: vgrStr = str("vgpr" + destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) if "A" in vgrStr: - tf32mod = Module() - tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr("G2LA+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr("G2LA+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr("G2LA+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) - localWriteCode.add(tf32mod) + emulationLocalWrite = F32XEmulationLocalWrite() + localWriteCode.add(emulationCvtLocalWrite()) LocalWriteX = tP["localWriteInstruction"].getInst(isHigh16Bits) if numBlocks == 1: diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 2c2944e2fb..692653f768 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -2020,9 +2020,11 @@ def calSwizzleK(state, tc): if state["UseF32XEmulation"]: if isaInfoMap[isa].archCaps["HasF32XEmulation"]: if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1 : + print("carson: reject1 " + str(state["VectorWidthA"]) + " " + str(state["VectorWidthB"])) reject(state, "Missing implementation for F32X Emulation VW>1") break if depthU != 16: + print("carson: reject2") reject(state, "Missing implementation for F32X Emulation DepthU!=16") break else: From 6dd97d56e87c041e6c587ef117841bfc0c9c5e9a Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Tue, 15 Apr 2025 18:00:49 -0500 Subject: [PATCH 02/18] tf32 support for different vector widths and depthu --- tensilelite/Tensile/SolutionStructs/Solution.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 692653f768..7ecf3e4cfb 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -2016,18 +2016,8 @@ def calSwizzleK(state, tc): reject(state, printRejectionReason, "Not implement DTVSM with VW>1") break - # f32 emulation currently only supports a limited set of solutions if state["UseF32XEmulation"]: - if isaInfoMap[isa].archCaps["HasF32XEmulation"]: - if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1 : - print("carson: reject1 " + str(state["VectorWidthA"]) + " " + str(state["VectorWidthB"])) - reject(state, "Missing implementation for F32X Emulation VW>1") - break - if depthU != 16: - print("carson: reject2") - reject(state, "Missing implementation for F32X Emulation DepthU!=16") - break - else: + if not isaInfoMap[isa].archCaps["HasF32XEmulation"]: reject(state, "Missing emulation for F32X") break From e50d7629eb254b461c0e3f3feaf1a201de5190b0 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Thu, 17 Apr 2025 20:08:38 -0500 Subject: [PATCH 03/18] tf32 lds debug --- .../Tensile/Components/F32XEmulation.py | 100 ++++++++++++++---- tensilelite/Tensile/Components/LocalRead.py | 7 +- tensilelite/Tensile/KernelWriterAssembly.py | 5 +- .../Tensile/SolutionStructs/Solution.py | 2 +- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 2aa0073a86..7bbef580ac 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -45,40 +45,85 @@ class F32XEmulationCvtLocalWrite(F32XEmulation): def __call__(self): tf32mod = Module() tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) + # From: + # + # 0: G2LA+0 + # 1: G2LA+1 + # 2: G2LA+2 + # 3: G2LA+3 + # + # To: + # old + # 0: [0low, 0high] + # 1: [1low, 1high] + # 2: [2low, 2high] + # 3: [3low, 3high] + # + # + # 0: [0high, 1high] + # 1: [2high, 3high] + # 2: [0low, 1low] + # 3: [2low, 3low] + # + + + # high bits + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # low bits tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr("G2LA+0"), src1=vgpr("Cvt+8"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr("G2LA+1"), src1=vgpr("Cvt+9"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr("G2LA+2"), src1=vgpr("Cvt+8"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + + tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VMovB32(dst=vgpr("G2LA+1"), src=vgpr("Cvt+1"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+3"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+5"))) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) return tf32mod class F32XEmulationCvtLocalRead(F32XEmulation): asmCaps = {"HasMFMA_xf32": True} - def __call__(self): + def __call__(self, LocalReadX): tf32mod = Module() - tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? + # tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) + # tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) + # tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) + # tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) + # tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) return tf32mod @@ -93,6 +138,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v bStart = re.search(r'v\[vgpr(.*?):', str(src0)).group(1) tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) tf32mod.add(TextBlock("/*bf16AHigh*/\n")) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) if not kernel["EnableF32XEmulationLds"]: tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) @@ -126,8 +172,18 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v #todo: working impl using in situ cvt cmd. lds currently some kernels are failing if kernel["EnableF32XEmulationLds"]: - tf32mod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2)) + tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) + (src0, src1) = (vgpr("Cvt+6",2), vgpr(aStart,2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) else: tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+4",2)) @@ -144,4 +200,4 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(TextBlock(str("label_tf32_") + str(F32XEmulationMFMA.dbgCounter) + ":\n")) F32XEmulationMFMA.dbgCounter += 1 - return tf32mod \ No newline at end of file + return tf32mod \ No newline at end of file diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index 07f96e2bb9..757207f1d6 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -22,7 +22,7 @@ # ################################################################################ -from rocisa.code import Module +from rocisa.code import Module, TextBlock from rocisa.container import DSModifiers, vgpr, sgpr, SDWAModifiers, VOP3PModifiers from rocisa.enum import SelectBit from rocisa.instruction import SMovB32, SWaitCnt, VOrB32, VPermB32, VLShiftLeftOrB32, \ @@ -628,6 +628,9 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"] and tP["isA"] and tc == "A": tf32mod = F32XEmulationCvtLocalRead() - localReadCode.add(tf32mod()) + # localReadCode.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # testMod = imod.add(Module("Test")) + # testMod.add(SWaitCnt(lgkmcnt=0, comment="CheckValue1 wait for lds read")) + #localReadCode.add(tf32mod(LocalReadX)) return imod, pack diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 85879b3bb3..c7cddfb37e 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -9220,14 +9220,15 @@ def localWriteBody(tP): elif blockWidth == 0.5: localWriteCVTCode.add(VCvtF32toF16(dst=vgpr(destVgprPrefix + "+%u+%u"%(g2lIdxTmp, vi * 2 + interOffset)), src=vgpr(vgprTmp+1), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1), comment="Convert to FP16")) self.vgprPool.checkIn(vgprTmp) + None else: printExit("Unsupported combination DataType%s (%s) -> DataType (%s)"%(tc, kernel["ProblemType"]["DataType%s"%tc].toChar(), kernel["ProblemType"]["DataType"].toChar())) if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: vgrStr = str("vgpr" + destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) if "A" in vgrStr: - emulationLocalWrite = F32XEmulationLocalWrite() - localWriteCode.add(emulationCvtLocalWrite()) + emulationLocalWrite = F32XEmulationCvtLocalWrite() + localWriteCode.add(emulationLocalWrite()) LocalWriteX = tP["localWriteInstruction"].getInst(isHigh16Bits) if numBlocks == 1: diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 7ecf3e4cfb..ce96dcc059 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -891,7 +891,7 @@ def assignDerivedParameters( # the keys "EnableF32XdlMathOp" and "F32XdlMathOp". state["EnableF32XdlMathOp"] = False state["UseF32XEmulation"] = False #enable emulation for missing hardware support - state["EnableF32XEmulationLds"] = False + state["EnableF32XEmulationLds"] = True #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ From b31c813d6b1901604cb079e765396b278578495b Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Tue, 22 Apr 2025 13:23:25 -0500 Subject: [PATCH 04/18] adding attribute check to issueLatency calls --- .../Tensile/Components/F32XEmulation.py | 117 +++++++++++++----- tensilelite/Tensile/Components/LocalRead.py | 2 +- tensilelite/Tensile/KernelWriter.py | 16 ++- 3 files changed, 95 insertions(+), 40 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 7bbef580ac..3b59386c29 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -38,28 +38,31 @@ from ..Component import F32XEmulation -import re +import re, types class F32XEmulationCvtLocalWrite(F32XEmulation): asmCaps = {"HasMFMA_xf32": True} + dbgCounter = 0 def __call__(self): tf32mod = Module() tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) + if (F32XEmulationCvtLocalWrite.dbgCounter == 0): + tf32mod.add(TextBlock(str("label_tf32lds_begin_") + str(F32XEmulationCvtLocalWrite.dbgCounter) + ":\n")) # From: # - # 0: G2LA+0 - # 1: G2LA+1 - # 2: G2LA+2 - # 3: G2LA+3 + # 0: G2LA+0 = 0, 4, 8, 12 + # 1: G2LA+1 = 1, 5, 9, 13 + # 2: G2LA+2 = 2, 6, 10, 14 + # 3: G2LA+3 = 3, 7, 11, 15 # # To: - # old + # # 0: [0low, 0high] # 1: [1low, 1high] # 2: [2low, 2high] # 3: [3low, 3high] # - # + # Carson: cannot do this, as it will break the 4 stride reassembly # 0: [0high, 1high] # 1: [2high, 3high] # 2: [0low, 1low] @@ -67,12 +70,12 @@ def __call__(self): # - # high bits - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # # high bits + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) # low bits tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) @@ -90,40 +93,88 @@ def __call__(self): tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + + # tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VMovB32(dst=vgpr("G2LA+1"), src=vgpr("Cvt+1"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+3"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+5"))) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + if (F32XEmulationCvtLocalWrite.dbgCounter == 0): + tf32mod.add(TextBlock(str("label_tf32lds_end_") + str(F32XEmulationCvtLocalWrite.dbgCounter) + ":\n")) + F32XEmulationCvtLocalWrite.dbgCounter += 1 - tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VMovB32(dst=vgpr("G2LA+1"), src=vgpr("Cvt+1"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+3"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+5"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) return tf32mod +class A: + FOO = "foo of A" + +def my_method(cls): + return (cls, cls.FOO) + + +def issueLatencyOp(self): + return + class F32XEmulationCvtLocalRead(F32XEmulation): asmCaps = {"HasMFMA_xf32": True} + dbgCounter = 0 def __call__(self, LocalReadX): tf32mod = Module() # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? - # tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) - # tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) - # tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) - # tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) - # tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) - # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) + tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + F32XEmulationCvtLocalRead.dbgCounter += 1 + tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) + tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #a = A() + #a.instance_patched = types.MethodType(my_method, a) + # a = VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1)) + # VMovB32.instance_patched = types.MethodType(my_method, a) + #types.MethodType(issueLatencyOp, op) + #op.issueLatency = types.MethodType(issueLatencyOp, op) + #op.issueLatency() + #tf32mod.add(op) + + + # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + + + + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + + + + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) + F32XEmulationCvtLocalRead.dbgCounter += 1 return tf32mod @@ -198,6 +249,6 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2)) - tf32mod.add(TextBlock(str("label_tf32_") + str(F32XEmulationMFMA.dbgCounter) + ":\n")) - F32XEmulationMFMA.dbgCounter += 1 + tf32mod.add(TextBlock(str("label_tf32_") + str(F32XEmulationMFMA.dbgCounter) + ":\n")) + F32XEmulationMFMA.dbgCounter += 1 return tf32mod \ No newline at end of file diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index 757207f1d6..7ad54fd778 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -631,6 +631,6 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): # localReadCode.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) # testMod = imod.add(Module("Test")) # testMod.add(SWaitCnt(lgkmcnt=0, comment="CheckValue1 wait for lds read")) - #localReadCode.add(tf32mod(LocalReadX)) + localReadCode.add(tf32mod(LocalReadX)) return imod, pack diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 87b2143028..09d553dacf 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1119,8 +1119,9 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): latencyLeft -= (tPA["localWriteInstruction"].issueLatency*2) readLeftLROPT = 0 for j in range(len(localReadItemsThisLoop)): - latencyLeft -= localReadItemsThisLoop[j].issueLatency()*2 - readLeftLROPT += 1 if latencyLeft >= 0 else 0 + if (hasattr(localReadItemsThisLoop[j], "issueLatency")): + latencyLeft -= localReadItemsThisLoop[j].issueLatency()*2 + readLeftLROPT += 1 if latencyLeft >= 0 else 0 # at least 1 instruction readLeftLROPT = max(readLeftLROPT,1) # evenly schedule localread with each mfma @@ -1160,9 +1161,11 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): latencyLeft -= readLeft * tPA["localWriteInstruction"].issueLatency * 2 else: readLeft = len(localReadItemsThisLoop) - latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) + if (hasattr(j, "issueLatency")): + latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) else: - latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) + if (hasattr(j, "issueLatency")): + latencyLeft -= sum(j.issueLatency()*2 for j in localReadItemsThisLoop) # force to schedule all remaining localreads before start to schedule localwrite. if mfmaIndex == self.states.sync1LdsMfmaIndex and oneBufferScheduling: @@ -1296,8 +1299,9 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): if self.states.numItersPLR and iteration >= isBarrier: readLeftLROPT = 0 for j in range(len(localReadItemsNextLoop)): - latencyLeft -= localReadItemsNextLoop[j].issueLatency()*2 - readLeftLROPT += 1 if latencyLeft >= 0 else 0 + if (hasattr(localReadItemsNextLoop[j], "issueLatency")): + latencyLeft -= localReadItemsNextLoop[j].issueLatency()*2 + readLeftLROPT += 1 if latencyLeft >= 0 else 0 # at least 1 instruction readLeftLROPT = max(readLeftLROPT,1) readLeftLREven = numReadsInst / (numMfmaPerIter - i) From 85b4fe16532594fc7b5609e892c3847f1e4433b6 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Tue, 22 Apr 2025 18:49:08 -0500 Subject: [PATCH 05/18] tf32 fix lds --- .../Tensile/Components/F32XEmulation.py | 72 +++++++++---------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 3b59386c29..52b46ad03e 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -57,10 +57,10 @@ def __call__(self): # # To: # - # 0: [0low, 0high] - # 1: [1low, 1high] - # 2: [2low, 2high] - # 3: [3low, 3high] + # 0: [0high, 0low] + # 1: [1high, 1low] + # 2: [2high, 2low] + # 3: [3high, 3low] # # Carson: cannot do this, as it will break the 4 stride reassembly # 0: [0high, 1high] @@ -72,9 +72,10 @@ def __call__(self): # # high bits # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) # low bits tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) @@ -93,10 +94,12 @@ def __call__(self): tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) @@ -129,50 +132,41 @@ def __call__(self, LocalReadX): tf32mod = Module() # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) - tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - #a = A() - #a.instance_patched = types.MethodType(my_method, a) - # a = VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1)) - # VMovB32.instance_patched = types.MethodType(my_method, a) - #types.MethodType(issueLatencyOp, op) - #op.issueLatency = types.MethodType(issueLatencyOp, op) - #op.issueLatency() - #tf32mod.add(op) - - - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - # tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - - - - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + #pack high bits + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #pack low bits + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - + # read format: + # 0: [0high, 1high] + # 1: [2high, 3high] + # 2: [0low, 1low] + # 3: [2low, 3low] + # - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 return tf32mod From cb8c3797cb35629735273a0ed07928ae6f36bf28 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Thu, 24 Apr 2025 01:39:45 -0500 Subject: [PATCH 06/18] tf32 fix for hipblaslt build --- tensilelite/Tensile/Components/F32XEmulation.py | 9 +++++---- tensilelite/Tensile/KernelWriterAssembly.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 52b46ad03e..e5b9c5b560 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -135,10 +135,11 @@ def __call__(self, LocalReadX): tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 - tf32mod.add(LocalReadX(dst=vgpr("Cvt+0"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=0))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+1"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=256))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+2"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=512))) - tf32mod.add(LocalReadX(dst=vgpr("Cvt+3"), src=vgpr("LocalReadAddrA"), ds=DSModifiers(na=1, offset=768))) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("Cvt+0"), src=vgpr("ValuA_X0_I0+0"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+1"), src=vgpr("ValuA_X0_I0+1"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+2"), src=vgpr("ValuA_X0_I0+2"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+3"), src=vgpr("ValuA_X0_I0+3"))) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) #pack high bits diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index c7cddfb37e..1a6e5062d5 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -40,6 +40,7 @@ from rocisa.macro import MacroVMagicDiv, PseudoRandomGenerator from . import CUSTOM_KERNEL_PATH from .TensileInstructions import SelectBit, \ +<<<<<<< HEAD SBranchIfZero, SBranchIfNotZero, DSInit, VCvtBF16toFP32, \ ArgumentLoader, bomb, RegisterPool, \ allocTmpGpr, allocTmpGprList, log2, \ @@ -73,6 +74,19 @@ VMulHIU32, VMulLOU32, VMulPKF32S, VMulU32U24, VNotB32, VOrB32, VPackF16toB32, \ VPrngB32, VReadfirstlaneB32, VSubF32, VSubI32, VSubU32, VXorB32 +======= + SBranchIfZero, SBranchIfNotZero, SMulInt64to32, DSInit, VCvtBF16toFP32, \ + ArgumentLoader, bomb, vectorStaticDivideAndRemainder, \ + scalarStaticRemainder, \ + scalarUInt32RegDivide, scalarUInt32DivideAndRemainder, \ + scalarStaticDivideAndRemainder, scalarStaticCeilDivide, sMagicDiv, staticMultiply, staticMultiplyAdd, \ + scalarStaticMultiply, \ + RegisterPool, allocTmpGpr, allocTmpGprList, \ + log2, ceilDivide, DataType, \ + dataTypeToMfmaInstTypePair, dataTypeNameAbbrevToInstType, \ + Assert +from .TensileInstructions.Instructions import * +>>>>>>> b29ed83f (tf32 fix for hipblaslt build) from .Component import Component from .Components.F32XEmulation import F32XEmulationMFMA, F32XEmulationCvtLocalWrite from .KernelWriterModules import * @@ -9220,7 +9234,6 @@ def localWriteBody(tP): elif blockWidth == 0.5: localWriteCVTCode.add(VCvtF32toF16(dst=vgpr(destVgprPrefix + "+%u+%u"%(g2lIdxTmp, vi * 2 + interOffset)), src=vgpr(vgprTmp+1), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1), comment="Convert to FP16")) self.vgprPool.checkIn(vgprTmp) - None else: printExit("Unsupported combination DataType%s (%s) -> DataType (%s)"%(tc, kernel["ProblemType"]["DataType%s"%tc].toChar(), kernel["ProblemType"]["DataType"].toChar())) From a08721b5291163568d9c57f88d7ceca2cc48785b Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Mon, 5 May 2025 16:15:29 -0500 Subject: [PATCH 07/18] tf32 lds fix for broken wave dims --- ...gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml | 17 ++-- .../Tensile/Components/F32XEmulation.py | 78 ++++++++++--------- tensilelite/Tensile/Components/LocalRead.py | 14 ++-- tensilelite/Tensile/KernelWriterAssembly.py | 25 ++---- 4 files changed, 63 insertions(+), 71 deletions(-) diff --git a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml index 0a44d8b8a1..5bca355119 100644 --- a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml +++ b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Ailk_Bjlk_S_MX_B_UserArgs.yaml @@ -89,10 +89,11 @@ AssertSummationElementMultiple: 1 AssignedDerivedParameters: true AssignedProblemIndependentDerivedParameters: true - BaseName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x1J7V_jobmP44OFztrsbXl-d3dSIK0CEdTi8QZcUodlVY= + BaseName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x1S2F3kkilrgK40SskFsdYEE0gPv9CMGaBoFPKzPf7I1Y= BufferLoad: true BufferStore: true CUCount: null + CUOccupancy: -1 ClusterLocalRead: 0 CodeObjectVersion: '4' ConvertAfterDS: false @@ -130,7 +131,7 @@ SupportUserGSU: true, UseUniversalArgs: true} Kernel: true KernelLanguage: Assembly - KernelNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_K1_MIWT1_1 + KernelNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA4_GRVWB1_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA1024_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA1_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG64_4_1 LDSTrInst: false LSCA: 64 LSCB: 16 @@ -188,6 +189,7 @@ MacroTileA: 64 MacroTileB: 16 MagicDivAlg: 2 + MathClocksUnrolledLoop: 0 MatrixInstB: 1 MatrixInstBM: 1 MatrixInstBN: 1 @@ -226,7 +228,7 @@ PackedC1IndicesX: [1] PrefetchGlobalRead: 1 PrefetchLocalRead: 0 - PreloadKernArgs: 0 + PreloadKernArgs: true ProblemType: Activation: false ActivationComputeDataType: 0 @@ -308,13 +310,14 @@ ScheduleIterAlg: 1 ScheduleLocalWrite: 1 SolutionIndex: 0 - SolutionNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_GSU1_GSUC0_GSUWGMRR0_K1_MIWT1_1_SU32_SUM0_SUS256_WGM8_WGMXCC1_WGMXCCGn1 + SolutionNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA4_GRVWB1_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA1024_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA1_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG64_4_1_WGM8_WGMXCC1_WGMXCCGn1 SourceSwap: 1 StaggerU: 32 StaggerUMapping: 0 StaggerUStride: 256 StorePriorityOpt: false StoreRemapVectorWidth: 0 + StoreSwapAddr: false StoreSyncOpt: 0 StoreVectorWidth: 1 StreamK: 0 @@ -339,7 +342,7 @@ UnrollMajorLDSMetadata: true Use64bShadowLimit: 1 UseDotInstruction: false - UseF32XEmulation: false + UseF32XEmulation: true UseInstOffsetForGRO: 0 UseSgprForGRO: -1 Valid: true @@ -375,9 +378,7 @@ tailLoopOptB: false - [2, 3, 0, 1] - - - [128, 128, 1, 128, 128, 128, 128, 128] - - [0, 0.19] - - - [2048, 2048, 1, 256, 2048, 2048, 2048, 2048] - - [0, 15.01] + - [0, 0.35] - null - null - DeviceEfficiency diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index e5b9c5b560..b7d6f9039e 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -29,10 +29,10 @@ VCmpClassF32, VOrB32, VPackF16toB32, \ VAndOrB32, VBfeU32, VLShiftLeftB16, SNop, VMed3F32, \ VCvtPkF32toBF16, VAndB32, \ - VMovB32, VLShiftLeftB32 + VMovB32, VLShiftLeftB32, VSubF32, MFMAInstruction from ..TensileInstructions import * from ..TensileInstructions import VCvtBF16toFP32 -from ..TensileInstructions.Instructions import * +#from ..TensileInstructions.Instructions import * from ..TensileInstructions import DataType, \ SaturateCastType, VSaturateCastInt @@ -43,7 +43,7 @@ class F32XEmulationCvtLocalWrite(F32XEmulation): asmCaps = {"HasMFMA_xf32": True} dbgCounter = 0 - def __call__(self): + def __call__(self, srcStart): tf32mod = Module() tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) if (F32XEmulationCvtLocalWrite.dbgCounter == 0): @@ -73,32 +73,32 @@ def __call__(self): # # high bits # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr("G2LA+0"), src1=vgpr("G2LA+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(srcStart), src1=vgpr(srcStart + "+1"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr("G2LA+2"), src1=vgpr("G2LA+3"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(srcStart + "+2"), src1=vgpr(srcStart + "+3"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) # low bits tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr("G2LA+0"), src1=vgpr("Cvt+8"))) + tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr(srcStart), src1=vgpr("Cvt+8"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr("G2LA+1"), src1=vgpr("Cvt+9"))) + tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr(srcStart + "+1"), src1=vgpr("Cvt+9"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr("G2LA+2"), src1=vgpr("Cvt+8"))) + tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr(srcStart + "+2"), src1=vgpr("Cvt+8"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr("G2LA+3"), src1=vgpr("Cvt+9"))) + tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr(srcStart + "+3"), src1=vgpr("Cvt+9"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+0"), src0=vgpr("Cvt+2"), src1=vgpr("G2LA+0"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+1"), src0=vgpr("Cvt+3"), src1=vgpr("G2LA+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+2"), src0=vgpr("Cvt+4"), src1=vgpr("G2LA+2"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("G2LA+3"), src0=vgpr("Cvt+5"), src1=vgpr("G2LA+3"))) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+0"), src0=vgpr("Cvt+2"), src1=vgpr(srcStart + "+0"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+1"), src0=vgpr("Cvt+3"), src1=vgpr(srcStart + "+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+2"), src0=vgpr("Cvt+4"), src1=vgpr(srcStart + "+2"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+3"), src0=vgpr("Cvt+5"), src1=vgpr(srcStart + "+3"))) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) @@ -128,37 +128,38 @@ def issueLatencyOp(self): class F32XEmulationCvtLocalRead(F32XEmulation): asmCaps = {"HasMFMA_xf32": True} dbgCounter = 0 - def __call__(self, LocalReadX): + def __call__(self, dstStart): tf32mod = Module() # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(SNop(waitState=4, comment="wait for ds_read")) tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("Cvt+0"), src=vgpr("ValuA_X0_I0+0"))) - tf32mod.add(VMovB32(dst=vgpr("Cvt+1"), src=vgpr("ValuA_X0_I0+1"))) - tf32mod.add(VMovB32(dst=vgpr("Cvt+2"), src=vgpr("ValuA_X0_I0+2"))) - tf32mod.add(VMovB32(dst=vgpr("Cvt+3"), src=vgpr("ValuA_X0_I0+3"))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr("Cvt+0"), src=vgpr(dstStart + "+0"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+1"), src=vgpr(dstStart + "+1"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+2"), src=vgpr(dstStart + "+2"))) + tf32mod.add(VMovB32(dst=vgpr("Cvt+3"), src=vgpr(dstStart + "+3"))) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) #pack high bits - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+0"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+1"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+1"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) #pack low bits - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(VMovB32(dst=vgpr("ValuA_X0_I0+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+2"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+2"), src=vgpr("Cvt+1"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(VMovB32(dst=vgpr(dstStart + "+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # read format: @@ -184,6 +185,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v bStart = re.search(r'v\[vgpr(.*?):', str(src0)).group(1) tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) tf32mod.add(TextBlock("/*bf16AHigh*/\n")) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) if not kernel["EnableF32XEmulationLds"]: tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) @@ -220,16 +222,20 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v if kernel["EnableF32XEmulationLds"]: tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) (src0, src1) = (vgpr("Cvt+6",2), vgpr(aStart,2)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) else: tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+4",2)) diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index 7ad54fd778..cbf594d989 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -240,6 +240,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): for vIdx in range(0, numVectorsPerTile): for eIdx in range(0, numReadsPerVector): valuiIdx = int(valufIdx) + baseValuiIdx = valuiIdx localReadCode = imod.add(Module("LocalRead%s Valu%u"%(tc,valuiIdx))) if needPack or numSplitMetadata: packCode = pack.add(Module("packCode")) @@ -622,15 +623,14 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): elif kernel["ProblemType"]["DataType"].isSingle(): localReadCode.add(writer.assert_eq( dbgVgpr, 1.0) ) + if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"] and tP["isA"] and tc == "A": + tf32mod = F32XEmulationCvtLocalRead() + dstStart = "Valu%s_X%u_I%u+%u"%(tc, bufferIdx, iui, baseValuiIdx) + localReadCode.add(tf32mod(dstStart)) + + # DTV case, do not return local read code. Return pack code only. if (tP["isA"] or tP["isB"]) and kernel["DirectToVgpr%s"%tc]: imod = Module("LocalReadDo%s_I%s (Empty)" % (tP["tensorChar"],iui)) - if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"] and tP["isA"] and tc == "A": - tf32mod = F32XEmulationCvtLocalRead() - # localReadCode.add(VMovB32(dst=vgpr("ValuA_X0_I0+0"), src=vgpr("Cvt+0"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_1))) - # testMod = imod.add(Module("Test")) - # testMod.add(SWaitCnt(lgkmcnt=0, comment="CheckValue1 wait for lds read")) - localReadCode.add(tf32mod(LocalReadX)) - return imod, pack diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 1a6e5062d5..444f4a1b64 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -40,7 +40,6 @@ from rocisa.macro import MacroVMagicDiv, PseudoRandomGenerator from . import CUSTOM_KERNEL_PATH from .TensileInstructions import SelectBit, \ -<<<<<<< HEAD SBranchIfZero, SBranchIfNotZero, DSInit, VCvtBF16toFP32, \ ArgumentLoader, bomb, RegisterPool, \ allocTmpGpr, allocTmpGprList, log2, \ @@ -74,19 +73,6 @@ VMulHIU32, VMulLOU32, VMulPKF32S, VMulU32U24, VNotB32, VOrB32, VPackF16toB32, \ VPrngB32, VReadfirstlaneB32, VSubF32, VSubI32, VSubU32, VXorB32 -======= - SBranchIfZero, SBranchIfNotZero, SMulInt64to32, DSInit, VCvtBF16toFP32, \ - ArgumentLoader, bomb, vectorStaticDivideAndRemainder, \ - scalarStaticRemainder, \ - scalarUInt32RegDivide, scalarUInt32DivideAndRemainder, \ - scalarStaticDivideAndRemainder, scalarStaticCeilDivide, sMagicDiv, staticMultiply, staticMultiplyAdd, \ - scalarStaticMultiply, \ - RegisterPool, allocTmpGpr, allocTmpGprList, \ - log2, ceilDivide, DataType, \ - dataTypeToMfmaInstTypePair, dataTypeNameAbbrevToInstType, \ - Assert -from .TensileInstructions.Instructions import * ->>>>>>> b29ed83f (tf32 fix for hipblaslt build) from .Component import Component from .Components.F32XEmulation import F32XEmulationMFMA, F32XEmulationCvtLocalWrite from .KernelWriterModules import * @@ -9237,12 +9223,6 @@ def localWriteBody(tP): else: printExit("Unsupported combination DataType%s (%s) -> DataType (%s)"%(tc, kernel["ProblemType"]["DataType%s"%tc].toChar(), kernel["ProblemType"]["DataType"].toChar())) - if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: - vgrStr = str("vgpr" + destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) - if "A" in vgrStr: - emulationLocalWrite = F32XEmulationCvtLocalWrite() - localWriteCode.add(emulationLocalWrite()) - LocalWriteX = tP["localWriteInstruction"].getInst(isHigh16Bits) if numBlocks == 1: if (paramList[1] >= 0x20000): @@ -9260,6 +9240,11 @@ def localWriteBody(tP): else: ds = DSModifiers(na=2, offset0=paramList[2], offset1=paramList[3]) writeInst = LocalWriteX(dstAddr=vgpr(lwa), src0=paramList[0], src1=paramList[1], ds=ds, comment=comment) + if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: + vgrStr = str(destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) + if "A" in vgrStr: + emulationLocalWrite = F32XEmulationCvtLocalWrite() + localWriteCode.add(emulationLocalWrite(vgrStr)) if self.do["LocalWriteCVT"]: localWriteCode.add(localWriteCVTCode) if self.do["LocalWrite%s"%tc]: From a3adb39895db8e459612722643d4a0866bb42cad Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Sat, 31 May 2025 11:53:34 -0500 Subject: [PATCH 08/18] support for tf32 16x16x32 mfma --- tensilelite/Tensile/Common/ValidParameters.py | 2 +- .../Tensile/Components/F32XEmulation.py | 151 +++++++++++------- tensilelite/Tensile/KernelWriter.py | 2 +- tensilelite/Tensile/KernelWriterAssembly.py | 8 +- .../Tensile/SolutionStructs/Solution.py | 17 +- 5 files changed, 111 insertions(+), 69 deletions(-) diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index e82a07349f..dfc14a2f8c 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -111,7 +111,7 @@ def makeValidMFMA(): [16, 16, 16, 1], [4, 4, 4, 16], ] + [[32, 32, 16, 1], [16, 16, 32, 1],] + [[16, 16, 64, 1],[32, 32, 32, 1]] - validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1], [16, 16, 16, 1]] + validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1], [16, 16, 16, 1], [16, 16, 32, 1]] validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1], [32, 32, 64, 1], [16, 16, 128, 1]] validMFMA["B8"] = validMFMA["F8"] validMFMA["F8B8"] = validMFMA["F8"] diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index b7d6f9039e..05f4f1bf2d 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -45,6 +45,7 @@ class F32XEmulationCvtLocalWrite(F32XEmulation): dbgCounter = 0 def __call__(self, srcStart): tf32mod = Module() + return tf32mod tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) if (F32XEmulationCvtLocalWrite.dbgCounter == 0): tf32mod.add(TextBlock(str("label_tf32lds_begin_") + str(F32XEmulationCvtLocalWrite.dbgCounter) + ":\n")) @@ -72,34 +73,34 @@ def __call__(self, srcStart): # # high bits # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(srcStart), src1=vgpr(srcStart + "+1"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(srcStart + "+2"), src1=vgpr(srcStart + "+3"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) # low bits tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+2"), src0=vgpr(srcStart), src1=vgpr("Cvt+8"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+3"), src0=vgpr(srcStart + "+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+4"), src0=vgpr(srcStart + "+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VSubF32(dst=vgpr("Cvt+5"), src0=vgpr(srcStart + "+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+0"), src0=vgpr("Cvt+2"), src1=vgpr(srcStart + "+0"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+1"), src0=vgpr("Cvt+3"), src1=vgpr(srcStart + "+1"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+2"), src0=vgpr("Cvt+4"), src1=vgpr(srcStart + "+2"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(srcStart + "+3"), src0=vgpr("Cvt+5"), src1=vgpr(srcStart + "+3"))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # tf32mod.add(VMovB32(dst=vgpr("G2LA+0"), src=vgpr("Cvt+0"))) # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) @@ -115,13 +116,6 @@ def __call__(self, srcStart): return tf32mod -class A: - FOO = "foo of A" - -def my_method(cls): - return (cls, cls.FOO) - - def issueLatencyOp(self): return @@ -130,10 +124,11 @@ class F32XEmulationCvtLocalRead(F32XEmulation): dbgCounter = 0 def __call__(self, dstStart): tf32mod = Module() + return tf32mod # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(SNop(waitState=4, comment="wait for ds_read")) + #tf32mod.add(SNop(waitState=27, comment="wait for ds_read")) tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) @@ -160,7 +155,7 @@ def __call__(self, dstStart): tf32mod.add(VMovB32(dst=vgpr(dstStart + "+3"), src=vgpr("Cvt+2"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_0, src0_sel=SelectBit.WORD_0))) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(VMovB32(dst=vgpr(dstStart + "+3"), src=vgpr("Cvt+3"), sdwa=SDWAModifiers(dst_sel=SelectBit.WORD_1, src0_sel=SelectBit.WORD_0))) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # read format: # 0: [0high, 1high] @@ -178,75 +173,115 @@ class F32XEmulationMFMA(F32XEmulation): asmCaps = {"HasMFMA_xf32": True} #kernel = {"ProblemType": {"DataType": "F32XdlMathOp"}, "UseF32XEmulation"} dbgCounter = 0 - def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k, neg_flag): + # width = vgprPerInput (4 or 8) + def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k, width, neg_flag): tf32mod = Module() tf32mod.add(TextBlock("/*tf32 emulation*/\n")) aStart = re.search(r'v\[vgpr(.*?):', str(src1)).group(1) bStart = re.search(r'v\[vgpr(.*?):', str(src0)).group(1) + print("src0: {0} aStart: {1}".format(src0, aStart)) + print("src1: {0} bStart: {1}".format(src1, bStart)) tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) tf32mod.add(TextBlock("/*bf16AHigh*/\n")) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+0"), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) + numElementsPerIter = 4 + + #cvt0-1: Tmp + #cvt2-5: AHigh + #cvt6-9: Bhigh + #cvt12-15: ALow + #cvt16-19: BLow + + aHigh = "Cvt+2" + bHigh = "Cvt+6" + aLow = "Cvt+12" + bLow = "Cvt+16" + + for itr in range(int(width / numElementsPerIter)): + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+1"), src0=vgpr(aStart + "+2"), src1=vgpr(aStart + "+3"))) - tf32mod.add(TextBlock("/*bf16BHigh*/\n")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+2"), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+3"), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) - tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+0", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStart+"+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+0", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStart+"+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+4"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+1", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(aStart+"+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+1", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(aStart+"+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+5"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+2", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStart+"+0"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+2", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStart+"+1"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+6"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+8", src="Cvt+3", vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+8"), src0=vgpr(bStart+"+2"), src1=vgpr("Cvt+8"))) - tf32mod.add(VCvtBF16toFP32(dst="Cvt+9", src="Cvt+3", vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr("Cvt+9"), src0=vgpr(bStart+"+3"), src1=vgpr("Cvt+9"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr("Cvt+7"), src0=vgpr("Cvt+8"), src1=vgpr("Cvt+9"))) + offset = "+" + str(itr * 2) + aHigh1 = aHigh + offset + aHigh2 = aHigh + "+1" + offset + aLow1 = aLow + offset + aLow2 = aLow + "+1" + offset + bHigh1 = bHigh + offset + bHigh2 = bHigh + "+1" + offset + bLow1 = bLow + offset + bLow2 = bLow + "+1" + offset + tmp1 = "Cvt+0" + tmp2 = "Cvt+1" + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh1), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) + tf32mod.add(SNop(waitState=1, comment="1 wait s00tates for ds_read")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh2), src0=vgpr(aStart + "+2"), src1=vgpr(aStart + "+3"))) + tf32mod.add(TextBlock("/*bf16BHigh*/\n")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh1), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh2), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) + if kernel["EnableF32XEmulationLds"]: + None + #tf32mod.add(SNop(waitState=1020, comment="1 wait states for ds_read")) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + #tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + #(src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + #tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + + aStart = aStart + "+4" + bStart = bStart + "+4" #todo: working impl using in situ cvt cmd. lds currently some kernels are failing if kernel["EnableF32XEmulationLds"]: tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) (src0, src1) = (vgpr("Cvt+6",2), vgpr(aStart,2)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) else: tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+4",2)) + vgprSize = width / 2 + (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) - (src0, src1) = (vgpr("Cvt+6",2), vgpr("Cvt+0",2)) + (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr("Cvt+0",2)) + (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2)) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 09d553dacf..5ba5d3a089 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -4403,7 +4403,7 @@ def readWriteVectors(mat, vw, kernel): #align 64 bit vgprIdx = int((vgprIdx + 1) / 2) * 2 self.states.startVgprCvt = vgprIdx - vgprIdx += 9 # for vgpr serial id + vgprIdx += 20 # for vgpr serial id self.states.totalVgprs = max(vgprIdx, self.states.c.numVgprValu) if self.states.totalVgprs < 0 or self.states.totalVgprs > self.states.regCaps["MaxVgpr"]: diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 444f4a1b64..32c47e3f0c 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -5556,8 +5556,8 @@ def closeLoop(self, kernel, tPA, tPB, loopIdx, finalLoop, emitEndLabelOnly=False if noExit: # No exit. No dec code if decValue is 2 - if decValue == 2: - decCode = "" + # if decValue == 2: + decCode = "" condCode = "" nonFinalJumpNeeded = False if finalLoop: @@ -6769,8 +6769,10 @@ def mfmaIter(self, kernel, tPA, tPB, u, innerUnroll, vregSetIdx, unrollLoopIdx = acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)) acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)) emulation = F32XEmulationMFMA() + #mfma_1k = True imod.add(emulation(kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, variant, mfma_1k,\ - neg_flag=neg_flag)) + vgprPerInputA, neg_flag=neg_flag)) + imod.add(TextBlock("/*mfma ops: {0} {1}*/\n".format(str(src0), str(src1)))) else: imod.add(MFMAInstruction(instType=miInInstType, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=self.accVgprReadWriteIndex(kernel, (accStart+accStoreCIdx), (accEnd-accStart+1)), \ diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index ce96dcc059..7acb9855a2 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -891,7 +891,7 @@ def assignDerivedParameters( # the keys "EnableF32XdlMathOp" and "F32XdlMathOp". state["EnableF32XdlMathOp"] = False state["UseF32XEmulation"] = False #enable emulation for missing hardware support - state["EnableF32XEmulationLds"] = True + state["EnableF32XEmulationLds"] = False #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ @@ -1499,6 +1499,7 @@ def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: ldsPadB = state["LdsPadB"] optPadA = optPadB = lrvw readRegsA = readRegsB = lrvw * state["ProblemType"]["DataType"].numBytes() // 4 + print("calcLds, {0}, {1}".format(readRegsA, readRegsB)) if state["ProblemType"]["Sparse"]: if state["ProblemType"]["Sparse"] == 2: optPadB //= 2 @@ -1507,6 +1508,7 @@ def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: optPadA //= 2 readRegsA //= 2 if (not isaInfoMap[isa].asmCaps['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): + print("error1, {0}, {1}".format(readRegsA, readRegsB)) reject(state, "LocalReadVectorWidth results in attemping to read LDS larger than b128, reject") return if state["EnableMatrixInstruction"]: @@ -1586,6 +1588,7 @@ def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: if state["DirectToVgprB"]: ldsPadB = 0 + print("values: {0}, {1}, {2}".format(ldsPadA, ldsPadB, ldsPadM)) return ldsPadA, ldsPadB, ldsPadM def calcLdsBlockSizePerPad(lrvw: int) -> int: @@ -1707,9 +1710,9 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo if state["ProblemType"]["Sparse"] and state["MIInputPerThread"] * state["ProblemType"]["DataType"].numBytes() > 16: if state["LocalReadVectorWidth"] < state["MIInputPerThread"] // 2: reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"] // 2)) - elif not state["ProblemType"]["Sparse"] and not(state["ProblemType"]["DataType"].is8bitFloat() and (state["MatrixInstK"] == 64 or state["MatrixInstK"] == 128)): - if state["LocalReadVectorWidth"] < state["MIInputPerThread"]: - reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) + #elif not state["ProblemType"]["Sparse"] and not(state["ProblemType"]["DataType"].is8bitFloat() and (state["MatrixInstK"] == 64 or state["MatrixInstK"] == 128)): + #if state["LocalReadVectorWidth"] < state["MIInputPerThread"]: + # reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) if state["LocalReadVectorWidth"] > state["MIInputPerThread"] and not state["TransposeLDS"]: reject(state, printRejectionReason, "LocalReadVectorWidth require Transpose LDS") @@ -1865,8 +1868,8 @@ def calSwizzleK(state, tc): reject(state, printRejectionReason, "VWB * DataType.numBytes() > 16") # reject - GRVW too big - if (state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataTypeA"].numBytes()) > 16: - reject(state, printRejectionReason, "GRVWA * DataTypeA.numBytes() > 16") + #if (state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataTypeA"].numBytes()) > 16: + # reject(state, printRejectionReason, "GRVWA * DataTypeA.numBytes() > 16") if (state["GlobalReadVectorWidthB"] * state["ProblemType"]["DataTypeB"].numBytes()) > 16: reject(state, printRejectionReason, "GRVWB * DataTypeB.numBytes() > 16") @@ -2546,6 +2549,8 @@ def subCheckLdsBlockSizePerPad(tc, idx): state["NoLdsWriteCode"] = True # calculate ldsPad + result = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) + print("result: " + str(result)) state["LdsPadA"], state["LdsPadB"], state["LdsPadMetadata"] = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) if state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataType"].numBytes() == 32 and state["LdsPadA"] == 16 // state["ProblemType"]["DataType"].numBytes(): From 2b023bdab643b122bbb1be12df643186bf300aac Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Mon, 9 Jun 2025 16:09:21 -0500 Subject: [PATCH 09/18] tf32 16x16x32 debug --- tensilelite/Tensile/Components/F32XEmulation.py | 16 ++++++++++------ .../Tensile/Components/GlobalWriteBatch.py | 6 ++++++ tensilelite/Tensile/Components/LocalRead.py | 6 ++++++ tensilelite/Tensile/Components/StreamK.py | 1 + tensilelite/Tensile/KernelWriterAssembly.py | 16 ++++++++++++++-- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 05f4f1bf2d..566d342906 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -45,7 +45,6 @@ class F32XEmulationCvtLocalWrite(F32XEmulation): dbgCounter = 0 def __call__(self, srcStart): tf32mod = Module() - return tf32mod tf32mod.add(TextBlock("/*TF32 Emulation write lds*/\n")) if (F32XEmulationCvtLocalWrite.dbgCounter == 0): tf32mod.add(TextBlock(str("label_tf32lds_begin_") + str(F32XEmulationCvtLocalWrite.dbgCounter) + ":\n")) @@ -124,7 +123,6 @@ class F32XEmulationCvtLocalRead(F32XEmulation): dbgCounter = 0 def __call__(self, dstStart): tf32mod = Module() - return tf32mod # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) @@ -196,6 +194,8 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v aLow = "Cvt+12" bLow = "Cvt+16" + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + for itr in range(int(width / numElementsPerIter)): #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) @@ -210,6 +210,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v bLow2 = bLow + "+1" + offset tmp1 = "Cvt+0" tmp2 = "Cvt+1" + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) if not kernel["EnableF32XEmulationLds"]: tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh1), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) tf32mod.add(SNop(waitState=1, comment="1 wait s00tates for ds_read")) @@ -274,14 +275,17 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) vgprSize = width / 2 (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) + tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2)) diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index 36f50bf3fe..2de5820154 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -394,6 +394,7 @@ def _prolog(self, module: Module): module.add(VMovB32(dst=vgpr(bufferOOB), src="BufferOOB")) else: bufferOOB = None + print("carsonGlobalWriteBatchWriter1") #when factorDim = 1 the bias's gwvw is alwasy be 1. factor_gwvw = 1 if self.factorDim else self.ss.cfg.gwvw for elementIdx, element in enumerate(self.batchElements): @@ -417,6 +418,7 @@ def _prolog(self, module: Module): sumIdxGSUSYNC = self.ss.elementSumIdx[elementIdx] module.add(addrCalc.emitAddressSetupCode(self.kernel, self.tPB, self.ss, self.tmpVgpr, self.tmpS01, self.edge, self.beta, self.atomic, elementIdx, addrDVgpr)) + print("carsonGlobalWriteBatchWriter2") if self.edge: module.add(addrCalc.edgeProtectCode(self.kernel, self.edge, self.beta, self.atomic, mask, self.tmpSgpr)) @@ -427,8 +429,10 @@ def _prolog(self, module: Module): module.add(addrCalc.emitLdChange(self.kernel, self.ss, 'C', self.edge, self.beta, mask, bufferOOB, (elementIdx == 0), self.tmpVgpr, self.tmpSgpr, addrCVgpr, self.addrC, 0)) if dataBeta not in loadedDataBeta: if self.kernel["GroupLoadStore"]: + print("carsonGlobalWriteBatchWriter3") loadInputCode.add(self.parentWriter.readInput(self.kernel, self.ss, 'C', self.kernel["ProblemType"]["DestDataType"], addrCalc, vc0, data, self.gwvw, addrCVgpr, self.tmpS01)) else: + print("carsonGlobalWriteBatchWriter3.2", elementIdx) module.add(self.parentWriter.readInput(self.kernel, self.ss, 'C', self.kernel["ProblemType"]["DestDataType"], addrCalc, vc0, data, self.gwvw, addrCVgpr, self.tmpS01)) loadedDataBeta[dataBeta] = ceil(self.kernel["ProblemType"]["DestDataType"].numBytes() * self.ss.cfg.gwvw / 16) self.loadsBetaIssued += ceil(self.kernel["ProblemType"]["DestDataType"].numBytes() * self.gwvw / 16) @@ -439,8 +443,10 @@ def _prolog(self, module: Module): if dataE not in loadedDataE: loadOffset = int((self.kernel["ProblemType"]["ComputeDataType"].numRegisters() - self.kernel["ProblemType"]["DataTypeE"].numRegisters()) * self.ss.cfg.gwvw) if self.kernel["GroupLoadStore"]: + print("carsonGlobalWriteBatchWriter4") loadInputCode.add(self.parentWriter.readInput(self.kernel, self.ss, 'E', self.kernel["ProblemType"]["DataTypeE"], addrCalc, vc0, dataE + loadOffset, self.gwvw, addrEVgpr, self.tmpS01)) else: + print("carsonGlobalWriteBatchWriter4.2") module.add(self.parentWriter.readInput(self.kernel, self.ss, 'E', self.kernel["ProblemType"]["DataTypeE"], addrCalc, vc0, dataE + loadOffset, self.gwvw, addrEVgpr, self.tmpS01)) loadedDataE[dataE] = ceil(self.kernel["ProblemType"]["DataTypeE"].numBytes() * self.ss.cfg.gwvw / 16) self.loadsEIssued += ceil(self.kernel["ProblemType"]["DataTypeE"].numBytes() * self.gwvw / 16) diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index cbf594d989..0e2c625e9a 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -216,6 +216,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): numSplitMetadata = max(ceil((blockWidth * 4) // (kernel["MIInputPerThread%s"%tc] * tP["bpeDS"])) - 1, 0) if tP["isM"] else 0 valufIdx = 0 if enableLDSTr: + print("WEEEE1") numberMTilesPerWave = kernel["MIWaveTile"][tile01] highBits = 0 for tIdx in range(0, numberMTilesPerWave): @@ -237,6 +238,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): ds = DSModifiers(na=1, offset=paramList[1]) localReadCode.add(LocalReadX(dst=destVgpr, src=vgpr("LocalReadAddr%s"%tc), ds=ds, comment=comment)) else: + print("WEEEE2") for vIdx in range(0, numVectorsPerTile): for eIdx in range(0, numReadsPerVector): valuiIdx = int(valufIdx) @@ -545,11 +547,15 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): incOffset = rIdx * numElementPerRead * UnrollStride + incOffset offset_val = (incOffset + offset_val + tP["localReadOffset"]) * tP["bpeDS"] else: + print("offsetval: {0}, {1}, {2}", offset_val, numElementPerRead, UnrollStride) offset_val = (rIdx * numElementPerRead * UnrollStride + offset_val + tP["localReadOffset"]) * tP["bpeDS"] + print("offsetval2: ", offset_val) if (kernel["LdsBlockSizePerPad%s"%tc] != 0) and (kernel["LdsPad%s"%tc] != 0): offset_val = offset_val + (offset_val // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpeDS"] + print("offsetval3: ", offset_val, kernel["LdsBlockSizePerPad%s"%tc], kernel["LdsPad%s"%tc], tP["bpeDS"]) offset_val = offset_val + tP["localReadSwapByteOffset"] + # offset_val = offset_val * 4 #Carson: Debug if (kernel["DirectToLds%s" % tc] and \ kernel["GlobalReadVectorWidth%c"%tc] * tP["bpeDS"] > 4): # another address conversion for DirectToLds + NumLoadsCoalesced > 1 diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 6aa77c84dc..17d91a30db 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -1166,6 +1166,7 @@ def fixupBatch(self, writer, kernel, ss, batchIdx, edge, gwvw, \ # module.addComment1("WavefrontSize={}, WaveNum={}, storeWidth={}, bpeC={}".format(kernel["WavefrontSize"], WaveNum, storeWidth, writer.states.bpeCinternal)) module.add(SAddU32(dst=sgpr(tmpS01), src0=sgpr(tmpS01), src1=increment, comment="Inc sgpr offset")) + print("carsonStreamK") module.add(writer.readInput(kernel, ss, 'WS', kernel["ProblemType"]["ComputeDataType"], addrCalc, vc0, data, gwvw, addrCVgpr, sgpr(tmpS01))) loadsIssued += 1 diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 32c47e3f0c..683e65dbd1 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -107,6 +107,8 @@ class TailOptParams: firstLoop: int = 0 finalLoop: int = 0 +dbgCounter = 0 + ################################################################################ # Assembly Kernel ################################################################################ @@ -8256,6 +8258,7 @@ def globalReadBody(tP): # TODO: is it possible to load only hi16 when no in tail? (need to check INT8 too) datatype = kernel["ProblemType"]["DataType%s"%tc] if kernel["ConvertAfterDS"] else kernel["ProblemType"]["DataType"] isHigh16Bits = (datatype.isHalf() or datatype.isBFloat16()) and loopCnt%2==1 if not tP["isM"] else False + #Carson: global reads called here loadModule.add( self.chooseGlobalRead(kernel["BufferLoad"], \ bpl, destVgpr=destVgpr, \ addr0=vgpr(offsetVgpr), addr1=sgpr("Srd%s"%tc, 4), \ @@ -8263,6 +8266,7 @@ def globalReadBody(tP): glc=isGlc, slc=isSlc, nt=isNT, lds=isLds, \ hi16=isHigh16Bits , \ comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp))) + instOffset += 16 #Carson: debug 256b reads if unrollMirrorWithSoffset: codeMod = Module("mirrorIdx%u"%loopCnt) @@ -11790,6 +11794,7 @@ def bufferLoadImpl(soffset): # split into two dwordx4 loads. Second load offset is +0.5 bpl rv = Module("emulated _buffer_load_b256") dst = None if lds else vgpr(destVgpr, rpv//2) + print("emulatedb256 ", dst, addr0) rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ soffset=soffset, mubuf=mubuf, comment=comment)) mubuf2 = MUBUFModifiers(offen=True, offset12=int(offset + bpl/2), glc=glc, slc=slc, nt=nt, lds=lds) @@ -11798,8 +11803,10 @@ def bufferLoadImpl(soffset): elif isinstance(destVgpr, int): dst2 = int(destVgpr + int(rpv//2)) dst = None if lds else vgpr(dst2, rpv//2) - rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ - soffset=soffset, mubuf=mubuf2, comment=comment)) + print("emulatedb2562 ", dst, addr0) + rv.add(SWaitCnt(vmcnt=0, comment="")) + #rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ + # soffset=soffset, mubuf=mubuf2, comment=comment)) return rv elif bpl==64 and not lds: rv = Module("emulated _buffer_load_b512") @@ -11830,6 +11837,11 @@ def bufferLoadImpl(soffset): soffset=soffset, mubuf=mubuf4, comment=comment)) else: assert 0, "%s\nchooseGlobalRead: bad bpl %u"%(self.states.kernelName,bpl) + + global dbgCounter + rv.add(SWaitCnt(lgkmcnt=0, comment="")) + rv.add(TextBlock(str("label_gr_") + str(dbgCounter) + ":\n")) + dbgCounter += 1 # buffer_load offset field is 12-bit. # if offset >= 4096, use soffset instead From 159c19801570448e7543942c3b7cd0e12b1726cd Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Thu, 12 Jun 2025 14:38:03 -0500 Subject: [PATCH 10/18] adding absolute and relative error output to hipblaslt-bench client --- clients/include/allclose.hpp | 151 +- clients/include/argument_model.hpp | 23 +- clients/include/testing_matmul.hpp | 41 +- .../00_Final/cache.yaml | 105 + .../00_Final/source/ClientParameters.ini | 84 + .../00_Final/source/KernelHeader.h | 88 + .../source/Kernels/Cijk_SS_PostGSU16_VW1.cpp | 5078 +++++++ .../source/Kernels/Cijk_SS_PostGSU16_VW1.h | 217 + .../source/Kernels/Cijk_SS_PostGSU16_VW2.cpp | 9840 ++++++++++++ .../source/Kernels/Cijk_SS_PostGSU16_VW2.h | 217 + .../source/Kernels/Cijk_SS_PostGSU16_VW4.cpp | 12302 ++++++++++++++++ .../source/Kernels/Cijk_SS_PostGSU16_VW4.h | 217 + .../00_Final/source/ReductionTemplate.h | 151 + .../00_Final/source/TensileTypes.h | 763 + .../Kernels.so-000-gfx950-xnack+.hsaco | Bin 0 -> 168376 bytes .../Kernels.so-000-gfx950-xnack-.hsaco | Bin 0 -> 168368 bytes .../source/library/TensileLibrary.yaml | 180 + .../source/library/TensileLibrary_gfx950.co | Bin 0 -> 29072 bytes .../00_Final/source/memory_gfx.h | 597 + .../00_Final/source/tensile_bfloat16.h | 251 + .../00_Final/source/tensile_float8_bfloat8.h | 650 + .../source/tensile_float8_bfloat8_bc.h | 1215 ++ .../Data/00_Final.csv | 2 + .../Data/00_Final.yaml | 304 + 24 files changed, 32438 insertions(+), 38 deletions(-) create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack+.hsaco create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack-.hsaco create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary.yaml create mode 100755 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary_gfx950.co create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv create mode 100644 tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml diff --git a/clients/include/allclose.hpp b/clients/include/allclose.hpp index 790414ffec..884321db0e 100644 --- a/clients/include/allclose.hpp +++ b/clients/include/allclose.hpp @@ -48,8 +48,14 @@ /*! \brief compare the allclose error of two matrices hCPU & hGPU */ template -bool allclose(size_t* N, T* a, T* b, double atol, double rtol, bool equal_nan = false) +bool allclose(size_t* N, T* a, T* b, double atol, double rtol, + bool equal_nan, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { + bool success = true; for(size_t i = 0; i < *N; i++) { //Returning ture immediately if 2 elements are identical. @@ -60,14 +66,21 @@ bool allclose(size_t* N, T* a, T* b, double atol, double rtol, bool equal_nan = if(equal_nan && (std::isnan(a[i]) && std::isnan(b[i]))) continue; if(equal_nan && (std::isnan(a[i]) ^ std::isnan(b[i]))) - return false; + success = false; double error = std::abs(a[i] - b[i]); + double relError = error / std::abs(a[i]); double tolerance = atol + std::abs(rtol * b[i]); if(!(error <= tolerance)) - return false; + success = false; + absErrorAvg += error; + relErrorAvg += relError; + absErrorMax = std::max(error, absErrorMax); + relErrorMax = std::max(relError, relErrorMax); } - return true; + absErrorAvg = absErrorAvg / double(*N); + relErrorAvg = relErrorAvg / double(*N); + return success; } template < @@ -85,7 +98,12 @@ bool allclose_check_general(char allclose_type, T* hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg + ) { if(M * N == 0) return 0; @@ -109,7 +127,8 @@ bool allclose_check_general(char allclose_type, { for(auto& rtol : rtols) { - if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false)) + if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg)) { hipblaslt_atol = atol; hipblaslt_rtol = rtol; @@ -141,7 +160,11 @@ bool allclose_check_general(char allclose_type, T* hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -165,7 +188,8 @@ bool allclose_check_general(char allclose_type, { for(auto& rtol : rtols) { - if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false)) + if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg)) { hipblaslt_atol = atol; hipblaslt_rtol = rtol; @@ -197,7 +221,11 @@ bool allclose_check_general(char allclose_type, T* hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -221,7 +249,8 @@ bool allclose_check_general(char allclose_type, { for(auto& rtol : rtols) { - if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false)) + if(allclose(&size, hCPU_double.data(), hGPU_double.data(), atol, rtol, false, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg)) { hipblaslt_atol = atol; hipblaslt_rtol = rtol; @@ -254,7 +283,11 @@ bool allclose_check_general(char allclose_type, VEC&& hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -273,7 +306,8 @@ bool allclose_check_general(char allclose_type, } return allclose_check_general( - allclose_type, M, N, lda, hCPU_double, hGPU_double, hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU_double, hGPU_double, hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); } // For int8, we convert the results to int first @@ -285,7 +319,11 @@ bool allclose_check_general(char allclose_type, VEC&& hCPU, T* hGPU, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -304,7 +342,8 @@ bool allclose_check_general(char allclose_type, } return allclose_check_general( - allclose_type, M, N, lda, hCPU_int, hGPU_int, hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU_int, hGPU_int, hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); } /* ============== allclose check for strided_batched case ============= */ @@ -318,7 +357,11 @@ bool allclose_check_general(char allclose_type, T* hGPU, int64_t batch_count, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -327,7 +370,8 @@ bool allclose_check_general(char allclose_type, { auto index = i * stride_a; bool close = allclose_check_general( - allclose_type, M, N, lda, hCPU + index, hGPU + index, hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU + index, hGPU + index, hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); if(!close) return false; } @@ -345,7 +389,11 @@ bool allclose_check_general(char allclose_type, T* hGPU[], int64_t batch_count, double& hipblaslt_atol, - double& hipblaslt_rtol) + double& hipblaslt_rtol, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { if(M * N == 0) return 0; @@ -354,7 +402,8 @@ bool allclose_check_general(char allclose_type, { auto index = i; bool close = allclose_check_general( - allclose_type, M, N, lda, hCPU[index], hGPU[index], hipblaslt_atol, hipblaslt_rtol); + allclose_type, M, N, lda, hCPU[index], hGPU[index], hipblaslt_atol, hipblaslt_rtol, absErrorMax, absErrorAvg, + relErrorMax, relErrorAvg); if(!close) return false; } @@ -372,7 +421,11 @@ bool allclose_check_general(char allclose_type, int64_t batch_count, double& hipblaslt_atol, double& hipblaslt_rtol, - hipDataType type) + hipDataType type, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { switch(type) { @@ -386,7 +439,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_64F: return allclose_check_general(allclose_type, M, @@ -397,7 +454,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_16F: return allclose_check_general(allclose_type, M, @@ -408,7 +469,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_16BF: return allclose_check_general(allclose_type, M, @@ -419,7 +484,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8F_E4M3_FNUZ: return allclose_check_general(allclose_type, M, @@ -430,7 +499,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8F_E5M2_FNUZ: return allclose_check_general(allclose_type, M, @@ -441,7 +514,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); #ifdef ROCM_USE_FLOAT8 case HIP_R_8F_E4M3: return allclose_check_general(allclose_type, @@ -453,7 +530,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8F_E5M2: return allclose_check_general(allclose_type, M, @@ -464,7 +545,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); #endif case HIP_R_32I: return allclose_check_general(allclose_type, @@ -476,7 +561,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); case HIP_R_8I: return allclose_check_general(allclose_type, M, @@ -487,7 +576,11 @@ bool allclose_check_general(char allclose_type, static_cast(hGPU), batch_count, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); default: hipblaslt_cerr << "Error type in allclose_check_general" << std::endl; return false; diff --git a/clients/include/argument_model.hpp b/clients/include/argument_model.hpp index 1cc64860a7..f59611bed1 100644 --- a/clients/include/argument_model.hpp +++ b/clients/include/argument_model.hpp @@ -72,7 +72,11 @@ class ArgumentModel double cpu_us, double norm, double atol, - double rtol) + double rtol, + double absErrorMax, + double absErrorAvg, + double relErrorMax, + double relErrorAvg) { // requires enablement for frequency logging ArgumentModel_log_performance(name_line, val_line); @@ -154,6 +158,11 @@ class ArgumentModel else val_line << "," << rtol; } + if (absErrorMax != ArgumentLogging::NA_value) + { + name_line << ",absErrorMax,absErrorAvg,relErrorMax,relErrorAvg"; + val_line << "," << absErrorMax << "," << absErrorAvg << "," << relErrorMax << "," << relErrorAvg; + } } } } @@ -176,7 +185,11 @@ class ArgumentModel double cpu_us = ArgumentLogging::NA_value, double norm = ArgumentLogging::NA_value, double atol = ArgumentLogging::NA_value, - double rtol = ArgumentLogging::NA_value) + double rtol = ArgumentLogging::NA_value, + double absErrorMax = ArgumentLogging::NA_value, + double absErrorAvg = ArgumentLogging::NA_value, + double relErrorMax = ArgumentLogging::NA_value, + double relErrorAvg = ArgumentLogging::NA_value) { hipblaslt_internal_ostream name_list; hipblaslt_internal_ostream value_list; @@ -261,7 +274,11 @@ class ArgumentModel cpu_us, norm, atol, - rtol); + rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); if(archName != "") { diff --git a/clients/include/testing_matmul.hpp b/clients/include/testing_matmul.hpp index 2eeeeb6995..388420f87c 100644 --- a/clients/include/testing_matmul.hpp +++ b/clients/include/testing_matmul.hpp @@ -839,7 +839,11 @@ void check(hipStream_t stream, hipDataType To, hipDataType Tbias, hipDataType Taux, - hipDataType Tc) + hipDataType Tc, + double& absErrorMax, + double& absErrorAvg, + double& relErrorMax, + double& relErrorAvg) { // fetch GPU CHECK_HIP_ERROR(hipStreamSynchronize(stream)); @@ -1048,7 +1052,11 @@ void check(hipStream_t stream, num_batches[gemmIdx], hipblaslt_atol, hipblaslt_rtol, - To); + To, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); //TODO: confirm if allclose_check_assert is neccessary } } @@ -3343,6 +3351,11 @@ void testing_matmul_with_bias(const Arguments& arg, } } + double absErrorMax = 0; + double absErrorAvg = 0; + double relErrorMax = 0; + double relErrorAvg = 0; + if(!arg.timing) { for(size_t sol = 0; sol < heuristicResult.size(); sol++) @@ -3463,7 +3476,11 @@ void testing_matmul_with_bias(const Arguments& arg, To, Tbias, Taux, - Talpha); + Talpha, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } } } @@ -3866,7 +3883,11 @@ void testing_matmul_with_bias(const Arguments& arg, To, Tbias, Taux, - Talpha); + Talpha, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } #define argument_param \ @@ -3935,7 +3956,11 @@ void testing_matmul_with_bias(const Arguments& arg, cpu_time_used, hipblaslt_error, hipblaslt_atol, - hipblaslt_rtol); + hipblaslt_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } if(best_gpu_time > gpu_time_used) { @@ -3993,7 +4018,11 @@ void testing_matmul_with_bias(const Arguments& arg, cpu_time_used, best_norm, best_atol, - best_rtol); + best_rtol, + absErrorMax, + absErrorAvg, + relErrorMax, + relErrorAvg); } } diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml new file mode 100644 index 0000000000..5ba002b05f --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/cache.yaml @@ -0,0 +1,105 @@ +--- +CodeObjectFiles: [library/TensileLibrary_gfx950.co] +ConstantParams: + 1LDSBuffer: -1 + ActivationAlt: false + ActivationFuncCall: true + ActivationFused: true + AssertAIGreaterThanEqual: -1 + AssertAILessThanEqual: -1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 1 + BufferLoad: true + BufferStore: true + ClusterLocalRead: 1 + ConvertAfterDS: false + CustomKernelName: '' + DebugStreamK: 0 + DepthU: 32 + DirectToLds: false + DirectToVgprA: false + DirectToVgprB: false + DirectToVgprSparseMetadata: false + ExpandPointerSwap: true + ForceDisableShadowInit: false + GlobalReadPerMfma: 1 + GlobalReadVectorWidthA: 8 + GlobalReadVectorWidthB: -1 + GlobalSplitU: 1 + GlobalSplitUAlgorithm: MultipleBuffer + GlobalSplitUCoalesced: false + GlobalSplitUWorkGroupMappingRoundRobin: false + GroupLoadStore: false + InnerUnroll: 1 + InterleaveAlpha: 0 + KernelLanguage: Assembly + LDSTrInst: false + LdsBlockSizePerPadA: 512 + LdsBlockSizePerPadB: -1 + LdsBlockSizePerPadMetadata: 0 + LdsPadA: -1 + LdsPadB: -1 + LdsPadMetadata: 0 + LocalReadVectorWidth: 4 + LocalWritePerMfma: -1 + MIArchVgpr: false + MagicDivAlg: 2 + MatrixInstruction: [16, 16, 32, 1, 1, 1, 1, 1, 1] + MaxOccupancy: 40 + MbskPrefetchOpt: 0 + NoReject: false + NonTemporal: -1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NonTemporalD: 0 + NonTemporalE: 0 + NonTemporalMetadata: 0 + NonTemporalWS: 0 + NumElementsPerBatchStore: 0 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + OptNoLoadLoop: 1 + PrefetchGlobalRead: 1 + PrefetchLocalRead: 1 + PreloadKernArgs: true + ScheduleGlobalRead: 1 + ScheduleIterAlg: 3 + ScheduleLocalWrite: 1 + SourceSwap: 1 + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + StorePriorityOpt: false + StoreRemapVectorWidth: 0 + StoreSyncOpt: 0 + StoreVectorWidth: -1 + StreamK: 0 + StreamKAtomic: 0 + StreamKXCCMapping: 0 + SuppressNoLoadLoop: false + ThreadTile: [4, 4] + TransposeLDS: -1 + UnrollLoopSwapGlobalReadOrder: 0 + Use64bShadowLimit: 1 + UseInstOffsetForGRO: 0 + UseSgprForGRO: -1 + VectorStore: -1 + VectorWidthA: -1 + VectorWidthB: -1 + WaveSeparateGlobalReadA: 0 + WaveSeparateGlobalReadB: 0 + WaveSeparateGlobalReadMetadata: 0 + WaveSplitK: false + WavefrontSize: 64 + WorkGroup: [16, 16, 1] + WorkGroupMapping: 8 + WorkGroupMappingXCC: 1 + WorkGroupMappingXCCGroup: -1 + WorkGroupReduction: false +CustomKernelWildcard: false +CustomKernels: [] +ForkParams: {} +ParamGroups: [] +... diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini new file mode 100644 index 0000000000..9498e3d2aa --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ClientParameters.ini @@ -0,0 +1,84 @@ +library-file=/home/cbrownle/git/tests/build_small/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary.yaml +code-object=/home/cbrownle/git/tests/build_small/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/TensileLibrary_gfx950.co +results-file=/home/cbrownle/git/tests/build_small/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/../Data/00_Final.csv +performance-metric=DeviceEfficiency +problem-identifier=Contraction_l_Ailk_Bjlk_Cijk_Dijk +compute-input-type=Float +a-type=Float +b-type=Float +c-type=Float +d-type=Float +alpha-type=Float +beta-type=Float +f32-xdl-math-op=XFloat32 +activation-compute-type=Float +use-gradient=False +use-bias=0 +bias-source=3 +use-e=False +output-amaxD=False +use-scaleAB= +use-scaleCD=False +use-scaleAlphaVec=0 +swizzle-tensor-a=False +swizzle-tensor-b=False +icache-flush-args=False +sparse=0 +high-precision-accumulate=False +strided-batched=True +grouped-gemm=False +problem-size=16,16,1,32 +a-strides=-1,16,-1 +b-strides=-1,16,-1 +c-strides=-1,16,-1 +d-strides=-1,16,-1 +activation-type=None +activation-no-guard=False +activation-additional-args=2.0,2.0 +device-idx=0 +init-seed=0 +init-a=SerialIdx +init-b=Identity +init-c=Random +init-d=Zero +init-e=Zero +init-alpha=One +init-beta=Zero +init-bias=Random +init-scaleA=Two +init-scaleB=Two +init-scaleC=Two +init-scaleD=Two +init-scaleAlphaVec=Random +c-equal-d=False +prune-mode=PruneRandom +bounds-check=GuardPageFront +print-valids=False +print-max=4 +num-benchmarks=1 +num-elements-to-validate=-1 +num-enqueues-per-sync=10 +max-enqueues-per-sync=-1 +num-syncs-per-benchmark=1 +skip-slow-solution-ratio=0.0 +use-gpu-timer=True +hardware-monitor=False +num-warmups=5 +min-flops-per-sync=1 +sleep-percent=300 +perf-l2-read-hits=0.0 +perf-l2-write-hits=0.15 +perf-l2-read-bw-mul=2 +perf-read-efficiency=0.85 +csv-export-extra-cols=False +csv-merge-same-problems=False +log-level=Debug +max-workspace-size=13421772800 +PrintWinnersOnly=False +granularity-threshold=0.0 +pristine-on-gpu=True +library-update-file= +library-update-comment=False +use-user-args=False +rotating-buffer-size=0 +rotating-buffer-mode=0 diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h new file mode 100644 index 0000000000..d0e5553e90 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/KernelHeader.h @@ -0,0 +1,88 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef KERNEL_HEADER +#define KERNEL_HEADER + +#if Tensile_RUNTIME_LANGUAGE_OCL +#include +#else + +#include "TensileTypes.h" +#include + +#include "ReductionTemplate.h" +#include "memory_gfx.h" + + +__device__ inline int GenDot4(int a, int b, int c) +{ +#if(__hcc_workweek__ >= 19092) || __HIP_CLANG_ONLY__ + union + { + int32_t i; + char4 z; + } va, vb; +#else + typedef struct + { + int c0 : 8, c1 : 8, c2 : 8, c3 : 8; + } C4I8; + union + { + int32_t i; + C4I8 z; + } va, vb; +#endif + va.i = a; + vb.i = b; + +#if(__hcc_workweek__ >= 19092) || __HIP_CLANG_ONLY__ + return amd_mixed_dot(va.z, vb.z, c, true); +} +#else + return c + (vb.z.c3 * va.z.c3 + vb.z.c2 * va.z.c2 + vb.z.c1 * va.z.c1 + vb.z.c0 * va.z.c0); +} +#endif + +#endif // HIP + +typedef _Float16 tensile_half2 __attribute__((ext_vector_type(2))); +typedef _Float16 tensile_half; + +extern "C" __device__ tensile_half2 llvm_fma_v2f16(tensile_half2, + tensile_half2, + tensile_half2) __asm("llvm.fma.v2f16"); + +__device__ inline tensile_half2 + tensile_fmadd_half2(tensile_half2 multiplier, tensile_half2 multiplicand, tensile_half2 addend) +{ + tensile_half2 result; + result = llvm_fma_v2f16(multiplier, multiplicand, addend); + return result; +}; + +#endif diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp new file mode 100644 index 0000000000..806a8539cb --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.cpp @@ -0,0 +1,5078 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#include "Cijk_SS_PostGSU16_VW1.h" + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU16_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + } break; + case 14: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + } break; + case 13: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + } break; + case 12: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + } break; + case 11: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + } break; + case 10: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + } break; + case 9: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + } break; + case 8: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + } break; + case 7: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + } break; + case 6: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + } break; + case 5: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + } break; + case 4: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + } break; + case 3: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + } break; + case 2: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + } break; + case 1: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + } break; + default: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + } break; + } +#if defined(__gfx950__) +#endif + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW1( + argument_Cijk_SS_PostGSU16_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + } break; + case 14: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + } break; + case 13: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + } break; + case 12: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + } break; + case 11: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + } break; + case 10: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + } break; + case 9: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + } break; + case 8: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + } break; + case 7: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + } break; + case 6: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + } break; + case 5: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + } break; + case 4: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + } break; + case 3: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + } break; + case 2: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + } break; + case 1: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + } break; + default: + { + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[0]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[1]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[2]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[3]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[4]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[5]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[6]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[7]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[8]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[9]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[10]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[11]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[12]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[13]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[14]) + :); + asm __volatile__( + "v_add_f32 %0, %1, %0 \n\t" + : "+v"(accum[0]): "v"(temp[15]) + :); + } break; + } +#if defined(__gfx950__) +#endif + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU8_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + accum[0] += temp[4]; + accum[0] += temp[5]; + accum[0] += temp[6]; + accum[0] += temp[7]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW1( + argument_Cijk_SS_PostGSU8_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + accum[0] += temp[4]; + accum[0] += temp[5]; + accum[0] += temp[6]; + accum[0] += temp[7]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU4_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW1( + argument_Cijk_SS_PostGSU4_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + accum[0] += temp[2]; + accum[0] += temp[3]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW1* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU2_VW1 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW1); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW1); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW1); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW1( + argument_Cijk_SS_PostGSU2_VW1 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 1 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0]; + accum[0] += temp[1]; + + accum[0] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + } + + + result[0] = (float)accum[0]; + buffer_store(*(float1 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h new file mode 100644 index 0000000000..0e47c7fe6c --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW1.h @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#pragma once +#include +#include + +#include "KernelHeader.h" + + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU16_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU16_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW1( + argument_Cijk_SS_PostGSU16_VW1 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU8_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU8_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW1( + argument_Cijk_SS_PostGSU8_VW1 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU4_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU4_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW1( + argument_Cijk_SS_PostGSU4_VW1 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU2_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW1( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW1* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU2_VW1{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW1( + argument_Cijk_SS_PostGSU2_VW1 arg) +; diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp new file mode 100644 index 0000000000..283bcc9bf6 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.cpp @@ -0,0 +1,9840 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#include "Cijk_SS_PostGSU16_VW2.h" + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU16_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW2( + argument_Cijk_SS_PostGSU16_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[0]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[0].x), "v"(temp[0].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[1]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[1].x), "v"(temp[1].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[2]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[2].x), "v"(temp[2].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[3]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[3].x), "v"(temp[3].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[4]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[4].x), "v"(temp[4].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[5]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[5].x), "v"(temp[5].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[6]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[6].x), "v"(temp[6].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[7]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[7].x), "v"(temp[7].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[8]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[8].x), "v"(temp[8].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[9]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[9].x), "v"(temp[9].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[10]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[10].x), "v"(temp[10].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[11]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[11].x), "v"(temp[11].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[12]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[12].x), "v"(temp[12].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[13]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[13].x), "v"(temp[13].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[14]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[14].x), "v"(temp[14].y) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %1, %0 \n\t" + : "+v"(accumVec): "v"(temp[15]) +#else + "v_add_f32 %0, %2, %0 \n\t" + "v_add_f32 %1, %3, %1 \n\t" + : "+v"(accum[0]), "+v"(accum[1]): "v"(temp[15].x), "v"(temp[15].y) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU8_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW2( + argument_Cijk_SS_PostGSU8_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU4_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW2( + argument_Cijk_SS_PostGSU4_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW2* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU2_VW2 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW2); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW2); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW2); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW2( + argument_Cijk_SS_PostGSU2_VW2 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 2 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + buffer_store(*(float2 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h new file mode 100644 index 0000000000..cdc74bec85 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW2.h @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#pragma once +#include +#include + +#include "KernelHeader.h" + + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU16_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU16_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW2( + argument_Cijk_SS_PostGSU16_VW2 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU8_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU8_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW2( + argument_Cijk_SS_PostGSU8_VW2 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU4_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU4_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW2( + argument_Cijk_SS_PostGSU4_VW2 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU2_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW2( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW2* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU2_VW2{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW2( + argument_Cijk_SS_PostGSU2_VW2 arg) +; diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp new file mode 100644 index 0000000000..42acdd8d9d --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.cpp @@ -0,0 +1,12302 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#include "Cijk_SS_PostGSU16_VW4.h" + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU16_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU16_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 accumVec2(accum[2], accum[3]); + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; + accum[2] = accumVec2.x; + accum[3] = accumVec2.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW4( + argument_Cijk_SS_PostGSU16_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 16 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float2 accumVec(accum[0], accum[1]); + float2 accumVec2(accum[2], accum[3]); + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + int gsuRemain = (int)arg.gsu - NUM_GSU; + while(gsuRemain >= NUM_GSU) + { + gsuRemain -= NUM_GSU; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[15], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + } + switch(gsuRemain) + { + case 15: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[14], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + } break; + case 14: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[13], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + } break; + case 13: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[12], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + } break; + case 12: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[11], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + } break; + case 11: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[10], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + } break; + case 10: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[9], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + } break; + case 9: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[8], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + } break; + case 8: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + } break; + case 7: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + } break; + case 6: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + } break; + case 5: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + } break; + case 4: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + } break; + case 3: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + } break; + case 2: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + } break; + case 1: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + __builtin_amdgcn_sched_barrier(0); + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + __builtin_amdgcn_sched_barrier(0); + idxW += strideW; + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + } break; + default: + { + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[0].data.xy), "v"(temp[0].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[0].x), "v"(temp[0].y), "v"(temp[0].z), "v"(temp[0].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[1].data.xy), "v"(temp[1].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[1].x), "v"(temp[1].y), "v"(temp[1].z), "v"(temp[1].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[2].data.xy), "v"(temp[2].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[2].x), "v"(temp[2].y), "v"(temp[2].z), "v"(temp[2].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[3].data.xy), "v"(temp[3].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[3].x), "v"(temp[3].y), "v"(temp[3].z), "v"(temp[3].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[4].data.xy), "v"(temp[4].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[4].x), "v"(temp[4].y), "v"(temp[4].z), "v"(temp[4].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[5].data.xy), "v"(temp[5].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[5].x), "v"(temp[5].y), "v"(temp[5].z), "v"(temp[5].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[6].data.xy), "v"(temp[6].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[6].x), "v"(temp[6].y), "v"(temp[6].z), "v"(temp[6].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[7].data.xy), "v"(temp[7].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[7].x), "v"(temp[7].y), "v"(temp[7].z), "v"(temp[7].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[8].data.xy), "v"(temp[8].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[8].x), "v"(temp[8].y), "v"(temp[8].z), "v"(temp[8].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[9].data.xy), "v"(temp[9].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[9].x), "v"(temp[9].y), "v"(temp[9].z), "v"(temp[9].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[10].data.xy), "v"(temp[10].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[10].x), "v"(temp[10].y), "v"(temp[10].z), "v"(temp[10].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[11].data.xy), "v"(temp[11].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[11].x), "v"(temp[11].y), "v"(temp[11].z), "v"(temp[11].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[12].data.xy), "v"(temp[12].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[12].x), "v"(temp[12].y), "v"(temp[12].z), "v"(temp[12].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[13].data.xy), "v"(temp[13].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[13].x), "v"(temp[13].y), "v"(temp[13].z), "v"(temp[13].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[14].data.xy), "v"(temp[14].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[14].x), "v"(temp[14].y), "v"(temp[14].z), "v"(temp[14].w) +#endif + :); + asm __volatile__( +#if defined(__gfx950__) + "v_pk_add_f32 %0, %2, %0 \n\t" + "v_pk_add_f32 %1, %3, %1 \n\t" + : "+v"(accumVec), "+v"(accumVec2): "v"(temp[15].data.xy), "v"(temp[15].data.zw) +#else + "v_add_f32 %0, %4, %0 \n\t" + "v_add_f32 %1, %5, %1 \n\t" + "v_add_f32 %2, %6, %2 \n\t" + "v_add_f32 %3, %7, %3 \n\t" + : "+v"(accum[0]), "+v"(accum[1]), "+v"(accum[2]), "+v"(accum[3]): "v"(temp[15].x), "v"(temp[15].y), "v"(temp[15].z), "v"(temp[15].w) +#endif + :); + } break; + } +#if defined(__gfx950__) + accum[0] = accumVec.x; + accum[1] = accumVec.y; + accum[2] = accumVec2.x; + accum[3] = accumVec2.y; +#endif + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU8_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU8_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[2] += temp[4].z; + accum[3] += temp[4].w; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[2] += temp[5].z; + accum[3] += temp[5].w; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[2] += temp[6].z; + accum[3] += temp[6].w; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + accum[2] += temp[7].z; + accum[3] += temp[7].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW4( + argument_Cijk_SS_PostGSU8_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 8 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[4], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[5], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[6], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[7], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + accum[0] += temp[4].x; + accum[1] += temp[4].y; + accum[2] += temp[4].z; + accum[3] += temp[4].w; + accum[0] += temp[5].x; + accum[1] += temp[5].y; + accum[2] += temp[5].z; + accum[3] += temp[5].w; + accum[0] += temp[6].x; + accum[1] += temp[6].y; + accum[2] += temp[6].z; + accum[3] += temp[6].w; + accum[0] += temp[7].x; + accum[1] += temp[7].y; + accum[2] += temp[7].z; + accum[3] += temp[7].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU4_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU4_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW4( + argument_Cijk_SS_PostGSU4_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 4 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[2], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[3], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + accum[0] += temp[2].x; + accum[1] += temp[2].y; + accum[2] += temp[2].z; + accum[3] += temp[2].w; + accum[0] += temp[3].x; + accum[1] += temp[3].y; + accum[2] += temp[3].z; + accum[3] += temp[3].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW4* argsPtr, uint32_t gemm_count) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + uint32_t left = 0; + uint32_t middle; + uint32_t right = gemm_count; + uint32_t wiMiddle, wiLeft; + uint32_t targetP1 = id + 1; + while(left < right) + { + middle = (left + right) / 2; + wiMiddle = wiTablePtr[middle]; + if(wiMiddle < targetP1) + { + left = middle + 1; + wiLeft = wiMiddle; + } + else + right = middle; + } + id = id - wiLeft; + argument_Cijk_SS_GG_PostGSU2_VW4 arg; + int loadsInBytes = 0; + for(; loadsInBytes + 16 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW4); loadsInBytes += 16) + s_buffer_load(*((float4*) &arg + loadsInBytes/16), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 8 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW4); loadsInBytes += 8) + s_buffer_load(*((float2*) &arg + loadsInBytes/8), argsPtr+left-1, loadsInBytes); + for(; loadsInBytes + 4 <= sizeof(argument_Cijk_SS_GG_PostGSU2_VW4); loadsInBytes += 4) + s_buffer_load(*((float1*) &arg + loadsInBytes/4), argsPtr+left-1, loadsInBytes); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW4( + argument_Cijk_SS_PostGSU2_VW4 arg) +{ +/* hard-coded initial strides */ +#define strideD0I 1 +#define strideW0I 1 +#define strideC0I 1 +#define GLOBAL_D(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideD0I + (IDX1J)*arg.strideD1J + (IDXK)*arg.strideDK )) +#define GLOBAL_W(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideW0I + (IDX1J)*arg.strideW1J + (IDXK)*arg.strideWK )) +#define GLOBAL_C(IDX0I, IDX1J, IDXK) (( (IDX0I)*strideC0I + (IDX1J)*arg.strideC1J + (IDXK)*arg.strideCK )) +#define NUM_ELEMENT_LOAD 4 +#define NUM_GSU 2 + uint64_t id = hc_get_workitem_absolute_id(0); + + if (id*NUM_ELEMENT_LOAD >= (arg.size0I * arg.size1J * arg.sizeK)) + return; + + uint64_t id0, id1, id2; + id0 = (id % (arg.size0I/NUM_ELEMENT_LOAD)) * NUM_ELEMENT_LOAD; + id = id / (arg.size0I/NUM_ELEMENT_LOAD); + id1 = id % arg.size1J; + id = id / arg.size1J; + id2 = id % arg.sizeK; + id = id / arg.sizeK; + + uint64_t idxD = GLOBAL_D( (uint64_t)id0, id1, id2); + uint64_t idxW = GLOBAL_W( (uint64_t)id0, id1, id2); + uint64_t idxC = GLOBAL_C( (uint64_t)id0, id1, id2); + uint64_t strideW = 1 + (arg.size0I - 1) * strideW0I + (arg.size1J - 1) * arg.strideW1J + (arg.sizeK - 1) * arg.strideWK; + uint64_t strideWLimit = strideW * arg.gsu * sizeof(float); + float accum[NUM_ELEMENT_LOAD] = {0}; + float result[NUM_ELEMENT_LOAD]; + + float4 temp[NUM_GSU]; + buffer_load(temp[0], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + buffer_load(temp[1], arg.W, idxW * sizeof(float), 0, strideWLimit); + idxW += strideW; + + accum[0] += temp[0].x; + accum[1] += temp[0].y; + accum[2] += temp[0].z; + accum[3] += temp[0].w; + accum[0] += temp[1].x; + accum[1] += temp[1].y; + accum[2] += temp[1].z; + accum[3] += temp[1].w; + + accum[0] *= (float)arg.alpha; + accum[1] *= (float)arg.alpha; + accum[2] *= (float)arg.alpha; + accum[3] *= (float)arg.alpha; + + + if(arg.beta != (float)0){ + accum[0] += arg.beta * (float)arg.C[idxC+0]; + accum[1] += arg.beta * (float)arg.C[idxC+1]; + accum[2] += arg.beta * (float)arg.C[idxC+2]; + accum[3] += arg.beta * (float)arg.C[idxC+3]; + } + + + result[0] = (float)accum[0]; + result[1] = (float)accum[1]; + result[2] = (float)accum[2]; + result[3] = (float)accum[3]; + buffer_store(*(float4 *)result, arg.D, idxD * sizeof(float), 0); +} +#undef NUM_GSU +#undef NUM_ELEMENT_LOAD +#undef strideD0I +#undef strideW0I +#undef strideC0I +#undef GLOBAL_D +#undef GLOBAL_W +#undef GLOBAL_C diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h new file mode 100644 index 0000000000..d74cc47274 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/Kernels/Cijk_SS_PostGSU16_VW4.h @@ -0,0 +1,217 @@ +/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +#pragma once +#include +#include + +#include "KernelHeader.h" + + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU16_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU16_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU16_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU16_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU16_VW4( + argument_Cijk_SS_PostGSU16_VW4 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU8_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU8_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU8_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU8_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU8_VW4( + argument_Cijk_SS_PostGSU8_VW4 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU4_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU4_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU4_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU4_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU4_VW4( + argument_Cijk_SS_PostGSU4_VW4 arg) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_GG_PostGSU2_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_GG_PostGSU2_VW4( + uint32_t* wiTablePtr, void* deviceUserArgsPtr, argument_Cijk_SS_GG_PostGSU2_VW4* argsPtr, uint32_t gemm_count) +; + +struct __attribute__((__packed__)) argument_Cijk_SS_PostGSU2_VW4{ + float * D; + float * W; + float * C; + float alpha; + float beta; + unsigned int strideD1J; + unsigned int strideDK; + unsigned int strideW1J; + unsigned int strideWK; + unsigned int strideC1J; + unsigned int strideCK; + unsigned int size0I; + unsigned int size1J; + unsigned int sizeK; + unsigned int gsu; +}; + +extern "C" +__global__ void Cijk_SS_PostGSU2_VW4( + argument_Cijk_SS_PostGSU2_VW4 arg) +; diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h new file mode 100644 index 0000000000..ee61de51e9 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/ReductionTemplate.h @@ -0,0 +1,151 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once +#include "memory_gfx.h" +#include + +template +struct static_for +{ + template + __host__ __device__ constexpr void operator()(F const& f) const + { + if constexpr(Begin < End) + { + f(Begin); + static_for()(f); + } + } +}; + +constexpr size_t max(size_t a, size_t b) +{ + return (a > b) ? a : b; +} + +template +__device__ inline void + reductionKernel_ijk(DataTypeCompute* in, DataTypeOut* out, int m, int n, int strideJ) +{ + __shared__ DataTypeCompute block[MT0 * MT1 * VW]; + + int mod = hipThreadIdx_x % MT0; + int row = hipThreadIdx_x / MT0; + + int soffset = hipBlockIdx_x * MT0 * VW; + int voffset = mod * VW; + int idx = voffset + soffset; + + int soffsetBytes = soffset * sizeof(DataTypeCompute); + int soffsetBytes_bias = soffset * sizeof(DataTypeOut); + int num_records = strideJ * n * sizeof(DataTypeCompute); + int num_records_bias = m * sizeof(DataTypeOut); + + constexpr size_t sumLength = max((size_t)1, VW - 1); + DataTypeCompute sum[sumLength] = {0}; + if(idx + (VW - 1) < m) + { + for(int i = 0; i < n; i += MT1) + { + int currRow = row + i; + int rowStride = currRow * strideJ + voffset; + DataTypeCompute tmp[VW]; + static_for<0, VW>()([&](int vw) { + buffer_load( + tmp[vw], + reinterpret_cast(const_cast(in)), + (uint32_t)((rowStride + vw) * sizeof(DataTypeCompute)), + (uint32_t)(soffsetBytes), + num_records); + }); + static_for<0, VW>()([&](int vw) { sum[vw] += tmp[vw]; }); + } + static_for<0, VW>()([&](int vw) { block[(mod * MT1 + row) * VW + vw] = sum[vw]; }); + __syncthreads(); + if(hipThreadIdx_x < MT0) + { + for(int i = 1; i < MT1; i += 1) + { + static_for<0, VW>()( + [&](int vw) { sum[vw] += block[(i + hipThreadIdx_x * MT1) * VW + vw]; }); + } + // This is a 1D bias + static_for<0, VW>()([&](int vw) { + buffer_store( + (DataTypeOut)sum[vw], + reinterpret_cast(const_cast(out)), + (uint32_t)(voffset + vw) * sizeof(DataTypeOut), + (uint32_t)(soffsetBytes_bias), + num_records_bias); + }); + } + } + else if(idx < m) + { + for(int i = 0; i < n; i += MT1) + { + int currRow = row + i; + int rowStride = currRow * strideJ + voffset; + constexpr size_t tmpLength = max((size_t)1, VW - 1); + DataTypeCompute tmp[tmpLength]; + static_for<0, VW - 1>()([&](int vw) { + buffer_load( + tmp[vw], + reinterpret_cast(const_cast(in)), + (uint32_t)((rowStride + vw) * sizeof(DataTypeCompute)), + (uint32_t)(soffsetBytes), + num_records); + }); + static_for<0, VW - 1>()([&](int vw) { sum[vw] += tmp[vw]; }); + } + static_for<0, VW - 1>()([&](int vw) { block[(mod * MT1 + row) * VW + vw] = sum[vw]; }); + __syncthreads(); + if(hipThreadIdx_x < MT0) + { + + for(int i = 1; i < MT1; i += 1) + { + static_for<0, VW - 1>()( + [&](int vw) { sum[vw] += block[(i + hipThreadIdx_x * MT1) * VW + vw]; }); + } + // This is a 1D bias + static_for<0, VW - 1>()([&](int vw) { + buffer_store( + (DataTypeOut)sum[vw], + reinterpret_cast(const_cast(out)), + (uint32_t)(voffset + vw) * sizeof(DataTypeOut), + (uint32_t)(soffsetBytes_bias), + num_records_bias); + }); + } + } + else + { + static_for<0, VW>()([&](int vw) { block[(mod * MT1 + row) * VW + vw] = 0; }); + __syncthreads(); + } +} diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h new file mode 100644 index 0000000000..7be8412de7 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/TensileTypes.h @@ -0,0 +1,763 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef TENSILETYPES_H +#define TENSILETYPES_H +#include "tensile_bfloat16.h" + +#if (HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR == 2 && HIP_VERSION_PATCH > 42130) \ + || (HIP_VERSION_MAJOR == 6 && HIP_VERSION_MINOR >= 3)//tmp before gfx94 use hip f8 header + +// Using hip header for both NANOO and OCP data types +#if defined(__HIPCC__) +#include +#define TENSILELITE_FP8_TYPE_FNUZ HIP_FP8_TYPE_FNUZ +#define TENSILELITE_FP8_TYPE_OCP HIP_FP8_TYPE_OCP +#endif + +#include "tensile_float8_bfloat8.h" +#else + +#if !defined(HIP_FP8_TYPE_FNUZ) +#define TENSILELITE_FP8_TYPE_FNUZ 1 +#endif + +#if !defined(HIP_FP8_TYPE_OCP) +#define TENSILELITE_FP8_TYPE_OCP 0 +#endif + +#include "tensile_float8_bfloat8_bc.h" +#endif + +#include +#include +#include +#include +#include + +#include + +// OpenCL +#if Tensile_RUNTIME_LANGUAGE_OCL +#include "CL/cl.h" +#define TensileStatus cl_int +#define tensileStatusSuccess CL_SUCCESS +#define tensileStatusFailure -1 +#define tensileStatusAssertFailure -2 +#define TensileComplexFloat cl_float2 +#define TensileComplexDouble cl_double2 +#define TensileHalf cl_half + +// HIP +#else +#include +#define TensileStatus hipError_t +#define tensileStatusSuccess hipSuccess +// FIXME - steal hip error encodings since rocBLAS expects hip error codes to be +// returned... +#define tensileStatusFailure hipErrorUnknown +#define tensileStatusAssertFailure hipErrorRuntimeOther +#define TensileHalf _Float16 +inline std::ostream& operator<<(std::ostream& os, const _Float16& dt) +{ + os << (float)(dt); + return os; +} + +template +struct tensile_complex +{ + t x; + t y; + + __host__ __device__ tensile_complex() = default; + + constexpr __host__ __device__ tensile_complex(t real, t imag = 0) + : x{real} + , y{imag} + { + } + + template {}>::type* = nullptr> + tensile_complex& operator=(const u a) + { + x = a; + y = 0; + return (*this); + } +}; + +template +inline std::ostream& operator<<(std::ostream& os, const tensile_complex& data) +{ + if(data.y >= 0) + { + os << data.x << "+" << data.y << "i"; + } + else + { + os << data.x << data.y << "i"; + } + + return os; +} + +template +constexpr __host__ __device__ tensile_complex operator+(tensile_complex const& data) +{ + return data; +} + +template +constexpr __host__ __device__ tensile_complex operator-(tensile_complex const& data) +{ + return tensile_complex{-data.x, -data.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator+(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x + b.x, a.y + b.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator-(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x - b.x, a.y - b.y}; +} + +template +constexpr __host__ __device__ tensile_complex operator*(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{a.x * b.x - a.y * b.y, a.x * b.y + a.y * b.x}; +} + +template +constexpr __host__ __device__ tensile_complex operator/(tensile_complex const& a, + tensile_complex const& b) +{ + return tensile_complex{(a.x * b.x + a.y * b.y) / (b.x * b.x + b.y * b.y), + (a.y * b.x - a.x * b.y) / (b.x * b.x + b.y * b.y)}; +} + +template +constexpr __host__ __device__ tensile_complex& operator+=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a + b); +} + +template +constexpr __host__ __device__ tensile_complex& operator-=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a - b); +} + +template +constexpr __host__ __device__ tensile_complex& operator*=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a * b); +} + +template +constexpr __host__ __device__ tensile_complex& operator/=(tensile_complex& a, + tensile_complex const& b) +{ + return (a = a / b); +} +template +constexpr __host__ __device__ bool operator==(tensile_complex const& a, + tensile_complex const& b) +{ + return (a.x == b.x) && (a.y == b.y); +} + +using tensile_float_complex = tensile_complex; +using tensile_double_complex = tensile_complex; + +#define TensileComplexFloat tensile_float_complex +#define TensileComplexDouble tensile_double_complex +#endif // HIP + +#define TensileInt8x4 uint32_t +#define TensileInt32 int32_t + +/******************************************************************************* + * tensileSetup + ******************************************************************************/ +TensileStatus tensileSetup(); + +/******************************************************************************* + * tensileTeardown + ******************************************************************************/ +TensileStatus tensileTeardown(); + +/******************************************************************************* + * tensileCheckStatus + ******************************************************************************/ +#define tensileStatusCheck(RET) \ + { \ + TensileStatus tensileCheckStatusTmp = RET; \ + if(tensileCheckStatusTmp != tensileStatusSuccess) \ + { \ + fprintf(stderr, \ + "ERROR: TensileStatusFailure %i on line %u of %s\n", \ + tensileCheckStatusTmp, \ + __LINE__, \ + __FILE__); \ + abort(); \ + } \ + } + +// class ProblemDims +// - stores all dimensions of the problems (sizes and strides) +// TensileCreateLibrary.cpp will create a typedef for each specific problem, ie +// ProblemDims_Cijk_Ailk_Bljk_SB. +template +class ProblemDims +{ +public: + using SizeType = unsigned int; + + // Constructor accepts variable number of sizes: + template + explicit ProblemDims(Ts... args) + { + init(args...); + }; + + const SizeType strideD(int idx) const + { + return _dims[idx]; + } + + const SizeType strideC(int idx) const + { + return _dims[LastStrideD - FirstStride + idx]; + } + + const SizeType strideA(int idx) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + idx]; + } + + const SizeType strideB(int idx) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA + - FirstStride + idx]; + } + + const SizeType sizes(int idx = 0) const + { + return _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA + - FirstStride + LastStrideB - FirstStride + idx]; + } + + static int numSizes() + { + return NumSizes; + }; + + std::ostream& print(std::ostream& os) const + { + for(int i = 0; i < NumSizes; i++) + { + if(i != 0) + { + os << ", "; + }; + os << _dims[i]; + }; + return os; + }; + +private: + template + void init(T v) + { + _dims[NumSizes - I] = v; + } + + template + void init(T v, Ts... args) + { + _dims[NumSizes - I] = v; + init(args...); + } + +private: + // order: Dstride, Cstride, Astride, Bstrides, sizes + SizeType _dims[LastStrideD - FirstStride + LastStrideC - FirstStride + LastStrideA - FirstStride + + LastStrideB - FirstStride + NumSizes]; +}; + +// Base template for ProblemKey +// - stores the sizes and strides +// - supports hash generation and comparison for lookup +// TensileCreateLibrary.cpp will create a typedef for each specific problem, ie +// ProblemKey_Cijk_Ailk_Bljk_SB. +// Some templates below use a parm called ProblemKeyType which can be any of +// these generated types. +template +class ProblemKey +{ +public: + using SizeType = unsigned int; + + // Constructor accepts variable number of sizes: + template + explicit ProblemKey(Ts... args) + : _db(0) + { + init(args...); + } + + template + explicit ProblemKey(const ProblemDims& pdims) + : _db(0) + , _equalStrides(true) + { + for(int i = 0; i < NumSizes; i++) + { + _sizes[i] = pdims.sizes(i); + } + + _equalStrides + = ((pdims.strideD(0) == pdims.strideC(0)) && (pdims.strideD(1) == pdims.strideC(1))); + } + + bool operator<(const ProblemKey& p) const + { + if(p._equalStrides != this->_equalStrides) + return true; + for(int i = 0; i < NumSizes; i++) + { + if(p._sizes[i] < this->_sizes[i]) + return true; + else if(p._sizes[i] > this->_sizes[i]) + return false; + // if equal, continue to next index + } + return false; // get here if all indices are equal + }; + + bool operator==(const ProblemKey& p) const + { + if(p._equalStrides != this->_equalStrides) + return false; + for(int i = 0; i < NumSizes; i++) + { + if(p._sizes[i] != this->_sizes[i]) + return false; + } + return true; + }; + + size_t hash() const + { + size_t h = 0; + for(int i = 0; i < NumSizes; i++) + { + h ^= _sizes[i] + 0x9b9773e99e3779b9 + (h << 6) + (h >> 2); + } + return h; + } + + const SizeType sizes(int i) const + { + return _sizes[i]; + }; + + static int numSizes() + { + return NumSizes; + }; + + std::ostream& print(std::ostream& os) const + { + for(int i = 0; i < NumSizes; i++) + { + if(i != 0) + { + os << ", "; + }; + os << _sizes[i]; + }; + return os; + }; + +private: + template + void init(const SizeType& v) + { + _sizes[NumSizes - I - 1] = v; + } + + template + void init(const SizeType& v, Ts... args) + { + _sizes[NumSizes - I - 1] = v; + init(args...); + } + +private: + // Data members: + SizeType _sizes[NumSizes]; + bool _equalStrides; + uint32_t _db; +}; + +//------------- +// Distance Functions between two problem sizes - used to find nearest neighbor +// or closest solution Assumes p1 and p2 have same number of sizes +//------------- + +template +struct RatioDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 1.0; + for(int i = 0; i < p1.numSizes(); i++) + { + distance += ::fabs(::log(double(p1.sizes(i)) / double(p2.sizes(i)))); + } + return distance; + } +}; + +template +struct ManhattanDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 0; + for(int i = 0; i < p1.numSizes(); i++) + { + distance += ::fabs(double(p1.sizes(i)) - double(p2.sizes(i))); + } + return distance; + } +}; + +template +struct EuclideanDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + double distance = 0; + for(int i = 0; i < p1.numSizes(); i++) + { + if(p1.sizes(i) > p2.sizes(i)) + distance += pow(p1.sizes(i) - p2.sizes(i), 2); + else + distance += pow(p2.sizes(i) - p1.sizes(i), 2); + } + // distance = sqrt(distance); + return distance; + } +}; + +template +struct RandomDistance +{ + double operator()(const ProblemKeyType& p1, const ProblemKeyType& p2) const + { + return double(rand()); + } +}; + +// ProblemType +// Stores the different dimensions (free, index, batch) and other +// problem-specific info A single ProblemType may have many ProblemDims +// Combination of a ProblemType and ProblemDim defines a Problem +class ProblemType +{ +public: + ProblemType(const std::vector& indicesFree, + const std::vector& indicesSummation, + const std::vector& indicesBatch, + const std::vector& indexAssignmentsA, + const std::vector& indexAssignmentsB) + : _indicesFree(indicesFree) + , _indicesSummation(indicesSummation) + , _indicesBatch(indicesBatch) + , _indexAssignmentsA(indexAssignmentsA) + , _indexAssignmentsB(indexAssignmentsB) + { + } + + unsigned lastSummationIdx() const + { + return _indicesSummation.back(); + }; + unsigned free0Idx() const + { + return _indicesFree[0]; + }; + unsigned free1Idx() const + { + return _indicesFree[1]; + }; + bool isBatchIdx(unsigned idx) const + { + return std::find(_indicesBatch.begin(), _indicesBatch.end(), idx) != _indicesBatch.end(); + }; + unsigned numIndicesC() const + { + return _indicesFree.size() + _indicesBatch.size(); + }; + + const std::vector indexAssignmentsA() const + { + return _indexAssignmentsA; + } + const std::vector indexAssignmentsB() const + { + return _indexAssignmentsB; + } + +private: + const std::vector _indicesFree; + const std::vector _indicesSummation; + const std::vector _indicesBatch; + const std::vector _indexAssignmentsA; + const std::vector _indexAssignmentsB; +}; + +// These are properties of the problemDims and problemType which drive the +// 'assertions' +// - Solutions may be optimized so they only work for a subset of +// ProblemProperties - +// - At runtime, the ProblemProperties are computed for the given ProblemType +// and +// ProblemDims, and then checked against the Solution requirements. +// Must be checked by the runtime before launching the solution +struct ProblemProperties +{ + + // Constructor used in solution tables- + // See writeSolutionAndExactTable in TensileCreateLibrary - this constructor + // must be in-sync with the table written there. + ProblemProperties(unsigned summationElementMultiple, + unsigned free0ElementMultiple, + unsigned free1ElementMultiple, + int approxSize, + bool equalStrides, + bool allBatchAStridesAreZero, + bool allBatchBStridesAreZero) + : _summationElementMultiple(summationElementMultiple) + , _free0ElementMultiple(free0ElementMultiple) + , _free1ElementMultiple(free1ElementMultiple) + , _approxSize(approxSize) + , _equalStrides(equalStrides) + , _allBatchAStridesAreZero(allBatchAStridesAreZero) + , _allBatchBStridesAreZero(allBatchBStridesAreZero) + , _db(0) + { + const char* db = std::getenv("TENSILE_DB"); + if(db) + { + _db = strtol(db, nullptr, 0); + } + } + + // Constructor used to compute assertions for a specified problem size + template + ProblemProperties(const ProblemDimsType& pdims, const ProblemType* props) + : _db(0) + { + _summationElementMultiple = 1; // problem summation element multiple + auto sumSize = pdims.sizes(props->lastSummationIdx()); + if((sumSize & 0x7) == 0) + _summationElementMultiple = 8; + else if((sumSize & 0x3) == 0) + _summationElementMultiple = 4; + else if((sumSize & 0x1) == 0) + _summationElementMultiple = 2; + + auto free0Size = pdims.sizes(props->free0Idx()); + _free0ElementMultiple = 1; // problem free0 element multiple + if((free0Size & 0x7) == 0) + _free0ElementMultiple = 8; + else if((free0Size & 0x3) == 0) + _free0ElementMultiple = 4; + else if((free0Size & 0x1) == 0) + _free0ElementMultiple = 2; + + auto free1Size = pdims.sizes(props->free1Idx()); + _free1ElementMultiple = 1; // problem free1 element multiple + if((free1Size & 0x7) == 0) + _free1ElementMultiple = 8; + else if((free1Size & 0x3) == 0) + _free1ElementMultiple = 4; + else if((free1Size & 0x1) == 0) + _free1ElementMultiple = 2; + + bool allBelow1 = true; + bool anyBelow1 = false; + bool anyBelow4 = false; + for(int si = 0; si != pdims.numSizes(); si++) + { + if(!props->isBatchIdx(si)) + { + auto size = pdims.sizes(si); + if(size > 1) + allBelow1 = false; + if(size == 1) + anyBelow1 = true; + else if(size < 4) + anyBelow4 = true; + } + } + if(allBelow1) + _approxSize = 1; // really small + else if(anyBelow1 || anyBelow4) + _approxSize = 2; // one dim not big enough + else + _approxSize = 99; // big enough + + _equalStrides = true; + // Would need to fix for UseInitialStrides, if UseInitialStrides==1 then + // don't subtract 1 + for(int i = 0; i < props->numIndicesC() - 1; i++) + { + if(pdims.strideD(i) != pdims.strideC(i)) + { + _equalStrides = false; + break; + } + } + + _allBatchAStridesAreZero = 1; + int strideIdx = 0; + for(auto& idx : props->indexAssignmentsA()) + { + bool isb = props->isBatchIdx(idx); + // Would need to fix for UseInitialStrides + auto stride = strideIdx == 0 ? 1 : pdims.strideA(strideIdx - 1); + if(isb && stride != 0) + { + _allBatchAStridesAreZero = 0; + } + strideIdx++; + } + + _allBatchBStridesAreZero = 1; + strideIdx = 0; + for(auto& idx : props->indexAssignmentsB()) + { + bool isb = props->isBatchIdx(idx); + // Would need to fix for UseInitialStrides + auto stride = strideIdx == 0 ? 1 : pdims.strideB(strideIdx - 1); + if(isb && stride != 0) + { + _allBatchBStridesAreZero = 0; + } + strideIdx++; + } + }; + + // Returns True if this AsssertionProperties meet the requirements for the + // specified soluition (this object represents the 'Problem') + bool validForSolution(const ProblemProperties& solutionRequirements) const + { + bool rv + = (this->_summationElementMultiple >= solutionRequirements._summationElementMultiple) + && (this->_free0ElementMultiple >= solutionRequirements._free0ElementMultiple) + && (this->_free1ElementMultiple >= solutionRequirements._free1ElementMultiple) + && ((this->_approxSize) >= solutionRequirements._approxSize) + && ((this->_equalStrides) == solutionRequirements._equalStrides) + && ((this->_allBatchAStridesAreZero) >= solutionRequirements._allBatchAStridesAreZero) + && ((this->_allBatchBStridesAreZero) + >= solutionRequirements._allBatchBStridesAreZero); + +#if 0 + if(this->_db & 0x10) + { + if(!rv) + { +#define check_print(name, oper) \ + if(!(this->name oper solutionRequirements.name)) \ + { \ + std::cout << #name << "failed: !(" << this->name << #oper << solutionRequirements.name \ + << ")" << std::endl; \ + } + + std::cout << "Assertion failed:" << std::endl; + check_print(_summationElementMultiple, >=); + check_print(_free0ElementMultiple, >=); + check_print(_free1ElementMultiple, >=); + check_print(_approxSize, >=); + check_print(_equalStrides, ==); + check_print(_allBatchAStridesAreZero, >=); + check_print(_allBatchBStridesAreZero, >=); +#undef check_print + } + else + { + std::cout << "Satisfied asserts!" << std::endl; + } + } +#endif + + return rv; + } + + unsigned _summationElementMultiple; + unsigned _free0ElementMultiple; + unsigned _free1ElementMultiple; + int _approxSize; + bool _equalStrides; + bool _allBatchAStridesAreZero; // for problems: true if all batchA strides are + // 0. Solutions: true if all batchA strides + // must be 0 to run the kernel. Prob,Soln: x,0: + // ok: 0,1: FAILED_ASSERT, 1,1: OK . P>=S + bool _allBatchBStridesAreZero; // true if all batchB strides are 0 + uint32_t _db; +}; + +#endif diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack+.hsaco b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack+.hsaco new file mode 100644 index 0000000000000000000000000000000000000000..312a818b7e09dd47933eecd8c38d5c39ca9bed5c GIT binary patch literal 168376 zcmeFa3v^r8c_w;rZ~z>FAYKGPkfJDxf@q4OXo{w2N~3yMN3j&wwIxSM6<;^9Y{`ZZ z+LmQmQ8E$?B9bgSj@__bZklmhH_mjDv`w5xQzz{erA#}IwwYRTyQb~<&ehyb@AXQ1 zJF9a$Yuz=sW8UxE=lsOR5kL_jZATWU1?_Y8KIcDs|Br9K|NYbl@3_tM`KBzx`STl> z4WE0>V9LFI?Q^EOx+}!ENuve-$Bn2FL@eK(QmI#l|F26-Lfd_g(eh0>Ic!$o-P@9hiRd(Ed+6{OG~2-2bWl>Tcnw zUCvXM{?vDJ4<9^u=+XW6Pe1m+Cmibgk1xD&#}eQ8)WRFPm-xn~7v9+GykY9O`nH>^ zhmJh<shesq(_N+$#ASsr@mkwcFj-T&aj#}6G;k~O}o)Eo-E{|Pto!1TipJ!=2<74cMrANqEEb!7U$ zqYpiK;Gsibx$E$uM?Y{!-v_?A5VilNCqDJa{f|xmPBqFSpE&s1$divg{NQ6p9#L7^U(S>7 z+apgtap;Nt2S0_3J^aA_CypL?;1fT9#1}qy^EaIv{d38#o>svXnTY#jU zI|c8?>%zM=i{Rb&1@An>mj`+2#v!lSI9g~4R%uOLVFXvhqp}4w8%J6ryE-hDne~2Z z9MuOF^)!wY<>ev8`o@u>**J20jLpW;LT|Hd!Og}|c}U@EX!rhV9F4t`jUz>Td61WA zLY_PP)$9*047V&GSh<_y2TXpy<=p9%Y(e`#v!lSA8hsq z_qzj{&B1ba{CwGi9{lgyM!_0 za=YTXpH^dM=bb?cMR|E`ik`-iqB(NZ963^}C(6^f`13hmQV5 zA8sFd;L%M8_`^EJO`9Hi@c7m(quY)@ibYKuWp$&h)ber3B-cTHe?vp-Ae+swW;3kW z3{wk%_m_K}&1P7At2rAO%^qJKxDGU%Va+irj5%Go*0S7*JiO%brsf#cvPQG(o9(Zs z8MXpb{i_>Cie}@e**I!8j+%`lXP-;u>g=VD$=BI9V$XAId5xoFr6U{O!wXa$g>Ie{me_%cpsp}8+kn~y6Gh&0zhHrGK`+XP%?UtE0)7V2yqt-zXv z=F;yb2g{p-<=(+^cG;GWcXcl4RK%CpIBG8Ftm2lZO3$b~N8a4+(%kLx z4mpvl4=n0w9IXISylV?Oxt{YWIJxXYSj(^F=c{|GYtG*_=kJ>HcgvZ-`)%>Y=K8hV zJW*EzX6+(7QeLY{dU?&4$))tzR$!GmF=-c^9yfqV&7NViXV~l+Ugtf-1?(#?qpA~G zdly00-xXx_QePgpZg_XVwb|Zlc0t*tY%XSh5B!){ZSO68!4}q9*4%--fO&dTa+zy8 z?|FA1N$n|Yw)c2IL~~_W(?cHhKsz8}_mX!Om6y<6XL|}4u+P&XTRO5XHWg5-y|+wo zz5U$*SGMyH8i^WPhf8)ka zdHAr^&5z;9PY0e?(_XDYuNkaC@>jQ=x<$7jDY-{d-Ojq{!S5Ac3Q6Z7KO=nTapH+D z{5yqs=I8q!`F_tW?|Ba&+VBo3@HpFU_uA(-?wc_TJATQWVrk8vG>o%`c@~$JjfeLE zX21d^gCA9|AHM~kXaDNV>8YMWg@r&Wtiy!}7 z^D}y!+lv4DaM$6+Y(z+&t#R+kGyTssC!-6IBlxB9Bj4BI$G;n$K=IeTRDo-r#wy(1 zgx@%RWB9GdkAkOM__x^!po@;oZgB6nAf&D>_D8L8)mc48J*f33$2dzmVbRqmiBftk zH`FEhV_SiLw80kq*gxYRZGwN)eO4=3{HTxe*C`iT2-kp0AC3L{A2@zs|KTT(9NK^E z;Uh<%JTUD>9XvFB=qL`AmDn27spEzu=rbDS*z#DP@*n1Xx-zo-FgLQ7iMXUIBg+qS zA>S1bJCXbM=N>-1|A8lQZtleR{hJR>j&0sDdSL83x zf?lG{3$f+LnfY?OR-$rb`C(?h9Pd6{8CiaqnZM#;CsHfX_{8Q1#vizO%jm(a2OpT6 zm{`c7mK8Se7vjqw;D;-cvL1cFz!Bwz47l-1HhO&dD~cm}l9n-g z9$p|-euZDJ$e8w(ium#?{7SWqT@hb?Wsy89J<#)}6~9KFl`N~gQY+rkO#Ov&&aO;z zzwa36A}>{AfqJP1+sbJgIBJ)>2=4eRvI42`NBq6b_=a%`A+cHi>vP|qV#QeFk9E`! zYP>JsSCw9k_gNS0sPVph|2!+b8h@-uYkgndFWyq)Wkj>C`Byc*FZrMPsoL+8XSMP# z?;Cq+zt8V~pvLzl|L6DCexLQ`g&Ob6_lx^#zrU34PaUZJKIQlOSNXp2Xzlme4!COh z*>S1yx0w7F%73!P%ZOf5`AfCmU+nuLKjZnv-k1EJ|9tKDmy-YDFV=p4Df%y7XNIG_eFkFcey}Vg#g>X z|E|jWClJ3-{zC2d*=KkG@3X6@OE-V-~W%KL3j5{e{uuy>D*DFwazRH^b<8!*rT)KJ)xxuw}D9Sa|CEb94TA|C#xD zjDPseSF{h5V#$Ffo>O`cl^>6LTJCF12sq{h3SYn@pp9 z!$2;5(_pR`&E}alinJu2O>QP@&3%E%zD!p-87uftk33V21j%0buXQ~5FN=~ z=o!kTHV+!OGW&tyk) ziP%dg6S3j^6XajODkL{&jl`yYV{Qm5btfVwGu_>#UsLn7RTZ? z*%hKQ=%Hi0ebFBXoAt`e9ImAEN$U92x88{Re0C6F65%m7&fH*T7#G}RW}cL|&2HQl zGjmkpwz_dYVP=%g;2rZP-M9~$8R{&*VSdPs`)MA9k572?1#Ae$L6-nuy6Vwx7hiB{8eaL>LJ3AA9= z*1k0Ah899%o@qR#o(WF9`o>tGJ7qI$=}(!Y)A0k#PrK`ceSybjC~O)5KuEwW%p0FM zM?Cp~Q95U`j*g$WYzB=YCTW*lZ=DC$tFd`t{VrgA5h3RXiifQlm>%#fX%vF$$A9X9x31ri{<`%T+;Dl}uqbL3aZ zKdUgx3Iz=_fgUO*HO$aUCxIDH11FyOJKzMur;DZu>Nhr}ocVL39C&H~O~*kq-;gE5$$Y$HE0O7Dv8MUos%_ zAn@G4L>P6!hcFn3rg6ZBHeB`f!_!*=^R8FLko4ZQ`RnvX#qUL_ZXV())?sq zI7yBC_+!>2pXYTk5}hOsrpfzuAdcsqZ}OQyC_E|o^Mj7cUx0OklSea@%(ap}>VDQE z4EY@M$@duY`!?b&e^li&=!1=PSz}?ohjf*%PRq%QdtXdNRX!{8h3_Yv=be1Lb2z)AfG=U1&?l4w*GO0N+qH5>qbL(~qsyZGW-QYT_#*{Ap9ywjA~TLDtUr1^1HSWJ zUt4lpCT>pz)jOyJ$-88Dd&Yjz(O<*3SS)?jR}BAxy1!tj3`;%RVir?hbn0O!7RHe9 z9Q(+B_jXE0f7oz;u4__}LWW9~Ku8aOw^3b?wwjf+rJlEsuTQO6bF?W{r zi)|q@W)&zC6X-+!n<2atvXssk$g)a91I!2o^D~y6%LK#InV^$@e=F<{Fy+L*c(Ul| z^Bm&}2JJ6W@I!XRNTu$#Mr`PQ*wOt~SNFsAG3$f2qx-h)=)N6Rx*ymYE(BU^rTc*x z+6u8$!9t!5l-Wj}VRwK%=oLb{B@Vqp8n0V3ISc7w;A0Czv|VhZF$}^k$!p8E)?!)+ z&+t8lR;N8ix*=z)os&FT zt?67F=|d;kZeqyim`}dPklzm@KIl)XeA>Q*Wv#ZyB76_&{7zt%?c~M1FJ_V|pOyK7 zOkEoeseA$JVy3Cn4MTtZ9k7QEmK)bdSM)ozawn50lfy6Sd#i1lA%DEU=Uc=5mf4n4 zy59=E^Id-;y{$DBnNS#x3W;&WbaZ=b#L*p$t2w$Kb#*_IfxVMHbU)+heoHLMxzw`m zr@@yt&dL_&7=Hn_pmZN0bsuB(7XxVDqDHuA0XIbVsc*&B(u}pUtY6f9D`poc6VrD1 zZ^3S?R$J)|ZHH+VCTPvpaDK*)DI85|3EBkSZ%i9#8dYj{a~D<`|&~mvMa{R zy8lbC!&D>!-H$rDpK^6S8aWpHV8qe=NW{_oNL1;5XloSxnTXQ;P%K#p#xlSg=oZRM z95gb-b?83Aa9av`g*0BrdUC|i2-@Vt*$CSBVWcs%VUy%_(6_;2S__`xdkllCA_`wy z*bf6wL!FUHKCk4LN>7po)8zd~TL#ZN-&AsmK?iw{{IO0^`D=^hB#*&ZcP_?nLMP*- zg^5Bd;t=#D}%H;5i`ksoIW{Zpasc5%lB^=yOxws!_i)>59 zlM~cQ)Ro&)$&_;)P2<|pA82u<{%10V=zXX&%C!Pzy^lEh&A1D&0j2W@NuPB<5IWz| z5dr^xorwJ)9v8YAcNpb@l&&i5fKq~AW~BqOW16|yKwXOVAFlnIZOdd^qs3{XC0ko(eIavOYdAcU!h4q6?lO=k8;eDJ z%-aoE+k@u}^gWH1pcQTn&vb-aXTwHsuC)|>zGU=dP2;(9e&ciJ!oAsCskbl7zGc+# z7a7LG{$eTHxbb<*P-@FE9&nqSY}TlfH8z9F*LB%3L^}=VI;5tka+K#0F-bD{ei;< z^ZM!=W(xlpX2L0p-&-5Eh@KG!I;-jK7;u5X51s>7*Ge&J8r_**XH3IpX~%5cOb2c z<3gK9^IaQ%YarKo(}M zE|oGnvYoh}85+$+F#aWDUmcsfjYw=LAK92OX1a$|S_I=-90P0H$h2D(wzZ)=#JUjb zu{JgCl~Lnfq|b4&HjHP5x;Xyj#IubS$2gz4n(@2TyIG9$vEQ!FWz_ zs@C|+ZEE}lu%^ae*1F>_*(&2N{fmsh^tt0NeeU?n+Ky+{_)8Y;?{{s#BrWlV?Z4D3AcKPh8j?1$qy*b!6=qYo2KVe$<3VT}Fg<2&qcc*l3x z-|&v_T(9Fc&GtVB!JPL040?Xm=70aS-u|Cy&i^*&f1C5avi6}l|J$7ZZO;E<-A{A= zw>keS>mRPG`CogSbHkkTZNZ$czcsu~=6yLYtlIpE>T|!GJ7b%RV~ZB%ez`tL=6+4I z`%Pa|&Ha|Fo^05%a_j??tX_4^zQDp9u+^U{S$)}3e-?ATdoM>#&I9kg9J8H}^TE7N zM$lL2@5z;VhO^-wh9hn`n(aa#BC6(y_g+q8&X~`4#+@*OIb+^;r7HUtvW6Aw3(IR) z6WW?p%pWhTRmB{#%#mPSe0gmu=8+fXL@&u@Ts8as7_%hT*_kO#kl1T1>;1d1N(z z91o`qerK&=skwMev>4!x;rGrTAH5906|eRAW3DUZ{4ZAIncV+?FwQ=aKL6_jkJ&G{ za{q&YH5HByQ-)kO7Qk8~%8=JK<`aVnXaB>k?*0eRk8=JOVUY8`2s=3ci!jOgUxeWq zd!R3lx!?A-u|lA&zfkNO$#d^R*Uqs*V&_DmI53%)Ip2AAzL#^qob#QZm{9Y-t6~#{ zPUN8z^P@codoZuddEPYUd3!jgyA|`iHxB2zwqh1$l3qE^mgXM zrPWDVHyC-wrLab;^M--^%-~qAz4fJ&GjORv8?mwXA$8Lr)_{!V+iw~$B)!Gm|Byl) z*EY1{{WS8|&N=PY!93&A;CVaNcm>uD;OFdr2;2}hD({EY`zg#BGapgRjdNZ&hPm)4 z*7k8NQ+f^7kKHhkvw=ea->qssnCtmEP!1F8GyGVeQHm$CCA%*h#r}sF_CLh2{~^Ks z4?WuchdzD(!`{m+R^>W)`-Y)hX=p_4f2fB}r(H>>qft7Z`QV1>bXVr59cu*|Dwk(o zxnVlJE0mu;4;^4(#|_T@hkE4I-d{Lw5vD`V{sq<%tQ#1@Xt0tPYou9Ms67wP{sjx`jFN$5%HaE5rm82>YVDp=PXq%- zAyjqW0_qG<`f;i=im+qiKerta{}KAU_#dl@|6G>|{Kpfp(DC2!m+}8!x$A#G zKS}&YsPG@5+W&^oApRo+{`dCz+J*|TwowQF&Eb5<&Y?mEd*2c}#|y>YiTwP~kivP{ z@757RI`+KHkB_VP?$~&Nd(6^UZ?g*FDqtJ2vIAJT3SkBq-?3>Zm$`XZ;Xl)&NK^Rl zt`#B%692n_*-@1mRfH_ zT5EdpjLQJOXL<&+Y243@j_0DN0|fswTY8OX>u5f@xyP949aU+9|1rc7>!Wy|>rbO7 zW4d)%;eQM~j}rg=J@~zJQhj?p*I8ms@oLlqZal~B1HgaM@d49`|DAY_`~SLdPu!1f zNL$hM;as34nNzlgv^e%Al<_}XH~uF*_|LV8c6z7-0Is=Cy(WwDs#^{R_vqYwdr+ z-l&RkZ^PO5^vl4kaKyyA?UC~^AfpxgoMa8MZ>L!(hM!Y=P*mF(YjNjIQ(=!UhCOT8 z^Q7-d!7^*LCj~lY;Ft5kZi$a-4-uWyaZieA=fc>JQfz$=O+oCrX~FvOpgrwx2^x|{ zyJdX&JtNQS2x=6MJ*!vHaL6N3hUpTGKu_=w!c^eWOT_>$cBa^0OR%r%V>EndamAYQLFJfp!Y? zIRC|YFMdbjob}Jxvx~g@u>XzkwSy+-#C*UX%U53eTr%x!*fiDpXZCwbW?0>q%t+P; z4DqSH>+Z`|h<(-FmjkNryZdsm#ku!+=EaD!*k@!C*SJ@T_~FBPdE$l-`$vMN9ZsnB zf{FFWDfA<;ejoiptmZbk9@)S;WcEi}f_N79LEJO616{y^-EU${(8smOA=5s_{$v~* z##z1u$|wCveP1JGVZ;x=>+ORo^?%e`|5xw(_2s;EJ~7Zhof7xWsy*-j8TQlX*6E%) zzdeKY6zhBo``ZI;S+&-;X!qx1J9`VsomsWkw-_GD&-eB!{fn`0m(8l{WGq{;=9hc8 z6X@S1po?AW*w1t7vaWUgIj$9E-!6&19n<4Tmvz9&8?!k+n}9vZTHyGN=xbvA?@T}X zb}g*uhp^6fFqhceZ*;bR*5(mI()4w{D&3`}_N1dOjo;XZZ$rOPPbY44_qd~uPqdDx zJ?>m@96{Z!_PO8KkA3bVxr9m&=c$+M2O^I~0{iBe_axdUs;uB~5_{+qtF!pMbW+*{ ziS_-4#L<2e>+vqWfjuG4{&uuIBIw_-u1}(_vNp!Cw{resV(m6d)t`uO$3Crv{q4zq z)bI7|Z{K}6;i=#4c8pE+57fE8efQ<|mdgHuy>1{^8W^mbPAcJ{1FUFI)VqIs_hq}i zl8(X;+gA8twc83mtaY}+59rMyg zx90lw|8w2+f0p6@;V<>|KTZD+P5%$F2D<70VXnFUr@8*8x&EiQ{-^2x0lt5(%k@83 z_y550vqsndiw}r}_5Zr>2l#`i=KCSsD{Fj4a;1?`#RtRfd zFh<&rc+Q7LFfQDM^)NF-W4TV{|6v&8!V|fUn}&=|_Een$L1 zv@8D)21zY5)p_*KZbeibsVUxluYyz;Bi^JD5)p*j8m3%JVur?2VbAI<$w&GC=s_=nm9 zPT#N1@sH;CM|1q6IsVZc|F~YqKi>KMPtLw4XAhLV?@9JTaqm-tYa!hIOSSetC9wZV z_l>}@2Gu`seO{_-tQVEO1bhvb_akBd68lQ;xBW;})_+~G|0f2Yxz*Qz_2kqZHh29u z{i@3PZ{=f^{mwRL|IgpJ{eP_aN_p0TrMds7a{af7_1~j~IQsu;|BpG5=h|=f|JC~M zKAeA|opT~%0IL75t^byF-?H9Yor8jPV40govF1CQ=Q${y=<9RsR}|~R)!8Rl`;ESQ zj?ZTNjKlhGtmTSg|4%p1KfyZfF08A*5FWt(pWYnTeRuiC^Ico8|Hn;6cK z-{V;-QLKyP8mum?3!5>s+01IJ%N!fe`LOoOhc(|E7vs9^jx~5^OON40-e!76-Slui z#x-160}j9LSig=*QXcu{G{=0dCm9o_x!Xc26Lssp}P0{ba>W$w{IB67-j=|hdg5k z?ZA$7BYRrXSFQZC;B~-am!-QNM6P(W$lyrK;qb?tbYzqb&+=rTO!QR*#an7@wrmu5S>CoFG-lxxuB*qbqO@=z> znSPFvq&Wzh7~_<^t*IMvb~N_FDw=MZ>}6#d$29q_M8tmEHkn`LbbD|ss0!#5Y!I4K`T z_3eSZ!pBi-GFM7Aum{#N-r3U1J+RKW>C(@N-hH{lvzNBLmA;9F>)uP7_UwTTY%Gtz z){~#kj>>V8YFRb4D1U$Z^6gFay zu)@ZKi;dVTtgsP#gcUY!bg^-hi;XwBdx$q<4{`sg6Y&b0O6pVw}sQ z#)MPam@xen(RWFeJ;Xk?{}O)qBETNvWQVE7gHu?OCcX%2j|ac|J_#Ca%Qf;|v;7Cu ztl|G;w%PuBPuqX&14jWL&q0NeB)4lrhhosq1u1&bB6X`VpG4- z(E>kgn}*>3#7WcJe=6O5f5HFB#y+g8cm1D`W)|Pnr<5BtJ#iY z8ry&Log(>$9~6`iZ8wkq6UW!dmA+5J&k5h@MEj3x;W-DISe4AFbJ`Kt0^eqA14eWo zDD8UtZv}jyz*fb_Sv~C}+tY62cuo=R2{m_&cEq*gV`MRlZ2H}xzBhkO9$cR~uoyPH z^JztMs4!XF_a9TbsNWpLRg!}XCml0 zvR;m&O|TKZ!=meZax9a%kL=xq@8 ztA_&V0{hjWfKdoSHXLg*At(BLTZI8P464rZJKgw>6|`mzEUly8zKCy+Wmx8nAO5SA zKR@3Ai|vIW@U0YqpVRK1Y{2X*gy7pd4!Q}@jKbe_jQ+0s9Dmn_@uSy+;CJ3Ao-z80 z=h!w+wqb0+c>C;sn}24`_Pvd>?BOfW*P>{0e4#E{T`Ne7ex_F{U#nM;7X4PQRKC`% zAg#U?q_uVhY0^04Rf%?`gL^7i}132sh4%dsH84nhdb{A-O zqaLi6R?>sTq}2mjU5(K4U@>W}1}%uE{(O0`n6!F9tFsYW9xNuUHK3Jlgq8=3NvjXE zIvSzn!D7-{3tFj0XnC-hwE97-y%Aa-EGDgWpp|TdmIsSTD+^kQMre7kn6%b|R=g2f z9xNuU0nmyyLd%22q;&&mMH`{z!D7-H1g*A4XnC-hv^Id2Ka;839wQzs#-vTLSoD<$ zbw1yfVe2>SIbyY8bXk^hYA)l?6vD=5&;8!prMXt)*>mA9Sabi2@AO>wC2Q^;hQDIX zeN2Vs+&20h|Z*`;ehO5)-ZIs@o>h#t$N^hh(y}m~2-B_L8+D7S(R;SnBD80?q z>8)#&-dJ^d*+%KzR7sDvu^zOj*A3an_?7eqmP2pLmGo{{4!wyh=?#M3%^2%W25dEV zuHym6Os>{fzB%U5{Cl^rtZ)}Ji@xHy0^WyT8}lT>48ju#HzNFJ2!|1V8sR2{pF%i- z@HoO75poWG6d~u}HzVZwq%njXE4T?E#|p*~a-3ibLXNRdAXNE9Xc?usiBM_od4Fl{ z1HRJS1?I*0(KG)PZRjS}I%67w-A$}r+L%!Lb&j>PYvOFf*m#X%?Zh&$aiR(~5^sr% z8xRr~iI2oZ;v;d9_()tNJ`xv+kHkgdBXN=VNL(a75*LY{vGagq2)TqHgc7m1I8 ziytzdK^xSZ+th4xE{yFqaj?Ge>}w4NiCb)MHjG=IHs`g1gIkt?TU)B&7TcZDrXz+C zyNF@LE@Bw5ix@`iB8CyWh+)JoVi>WD7)I7*Uk*=)x(@e8`qP#^6bZpRTH*k5iKZ#;eGYX!f?mw`9qRq%%WX10@vJH#4d4zY%q zL#!d@5Nn7z#2R7_v4)sKtRdzQYlu0-8e$HyhL}UFA?6Tkh&jX>VvbMz!JC_wfe$xT!3U1hNqdA?L5v_)5F>~c z#0X*qF@jh@j38DJBZw8m2x0{>f>=R}AXX3~h!w;LVg)gRSV4>+RtQFzXpj6qTASp* zVn1ASZ0Y(OTcQne48LKU_OwZ^6^s~LhE0!Ev1yJkvCgM$)1GP9v}f8i?U{B>d!}8} zo@v*#XWBLGnRZQkrd`vXY1g!8+BNN&c1?SxUDKXv*J96K^`XxHf;soUVg9aZ$Jd4( z(@r=S*Rb7r=HspvJKnquyW3pF?l?Xobt!F-wnrPJ?a>Bld$d8?9&M1eM;oN=(FSRI zv_aY)ZIHG{8>H>g25Eb=LE0W|khVt~6x-AL_W!0?kG^AVZR$7Y!W*_1&wTi`Vt=E{ zu$R#)_QL)>>nYj|ZG|>NTcOR+R%kP{71|7Kg*HQ5q0P`%Xfw1G+6--lHbYyX&Cph8 zGqe@j3~hxrBewD^`=-BO&3(^6-}FUm?ll#fbHCH9w-)S%ddYdshV|UDj^tXgn;Vy* z=Qmc-bM{lEUZBoX*QxW=b?Q8IojOllr_NK?sq@rz>O6IwI!|4v&QsT^^VD_fJawHq zPhF?ZQ`f2UqU(P2ufC4?`&YDn)~{Z}^#k>YYo!|2OV4_#YemmTmZ6s;RrHd5E#fcr zlR8P=q)t*dsgu-A>LhiOI!WE6PEt3ilhjS>Bz2QIN!_GQQa7oS)J^Inb(16Ki+D>pbIl&t?=>_AJD{?x$8bNawJq*?4{S06o|+N*Eh<&) zD^z*B%09)6##0mf6b09*^W-i0NxqVw!}JTgzrBlE;OGEdAS^Ta$dPs}6p#60Sl`%fTW@<`vCREk8jb@mzTQR>DXrP8@t zJ|C*>7Y&-_b+sY~?opx~xW9;H=H4KdnfrQJX70&hnYo{aW#(QOmYMrlSZ3})VVSvq zgk|R550;txHdtovxnP;OUxH=kUI&)hM2Kw)Y7F!6UG(#J>O?qZg<~jt5Pqb(H<>2= zO{L&>ioA#*Zq<$OGs8Wuh+`NDbYbuF?l*(RQ-0EH4Oz$3yDfLCa}ig;zlR@s8w|mh zO~5GNtb@Wl_V!|@nDGOnbPn4k=8X8^%W$b!G>jSeN$~zP{RU7*&0muT*QXA6@u0r@ z^M-kTn6LQE^W2A8M2G`fjN)Od#+gTv9JX(r3l@WO=Ho^IXA936zTkPhV$4L$T+s{{ zf@To@9IS#FvU4-G-A6wX#7QfV0O8$l+Qw6%89T20NQC^e%Gay~|F34kG7LD2GD9z& z%$udB3+CI;{2lf}AbgsBO5r~Ad>r4|Wtgwd8$a)2tZU*|Whqk|98$;(uLB|dZoS#C7Ja1FJE=+6IN~LGh_MgL# z#i?_xp|f*13>Rg2ZpI%m3#mw?VC*yswr}V>UNdZS=zJeS-%hj8ZoF`A9)7Ogo`32t zltJ+>1i!!ZnHvF?oAB{q1W=s>3~U#Mzg!>V;JX6;6)gDEoiT9Mum!)o{^w?pqd6RK zY?xMX+6-BSq|wilf%|z=z7r(>FJYe>eb^POEr#+B8iDUzo{QvX-O?J&`EhOH++zBZ z@{?{Wz84JV{QMsL0Q>yO0-iJcR(Kj|MsR1iFcXZZbRYK9&4+LlrM<#1p^k&Ovbly%`|g^VfazL6v~pq_riYvtd00EsymiV;{B#MlOnxP z%Ail#))e2jFWEky)rxl6n#Dje|05mY9Wq(F(MhqXI-@23FEGh;iT(( zIOX~z?r?n-r(2xoIzu>{-ntcM)9=3A)#=2oQs>t1zTDH{+^XEu#J@Z9rGzhHj*S1jIh?|W%{ceY!L<39LR z+CBHf_h|Rr7r$D&KZyHY?fyR8uhH(gU%pSf=f3&1+CBHr_iOjuN54+H=YIP1ys7r@ zXSI0Z8a!EFG+CB2=JV^dv>iwraPMJR>Y3o4NZ#HP&v)+`&#&4uk?+|vnP0tUTfTSC z_WYVXx8(cwOy$?^xi#OvXGebBp4;--J-hPj_uQTz*mFnT+H+^VXxeDQy`b=kvL_C7 zpiS5DdWYK2pJ}npuDh_0AF|0n57>5WXWUN@sJM3M1mhBn`>B2vmw%D4}vRa`4{h;c29`#@5~wLqU37h>E`Mpax0y2ZFo#{JU~ z71s$JV_ceXKQXN0($F`?bujLq45_#d=pN%zjN3Y>;!@B<#w8hdb5_MAp_7b*Ustnh zOP`91Lq8c8Wn2Ph>VYp&=qlsjquA_f%c!^r^fsLiWB+h#36NB5ZM`z2PKS*a%Zx;F zW;~uV$H#N##6-@VoXnZqw&l$2+jHhEx8%&JshoN1tvPeYj+}YhZ8>w-uAF)M?K$&~ zJ92*PK{oHaGdDYbdd{|nRGrg-eaD|gUDttq$ImjvzT;<9XwIEQAI_d;#aLI$G55YNfqDI zQ;u&{@!M4V>ec1=78So;#rO7><3lR`78So{O*y_(#ZRgDzP@sNTE#m!w03PdzC*?D zP|x@GJ2Ffmj^!)GZ&UH>);aOXh4@`6KAUyo;|uY(tN8Wno%kr?ZS<9F_zEUJZJhU@ zbOiBildz8%#5MbfT?pAn>_*5w;wpsfBlaLa}Qw7I3zH`ar#)K2R^H z57Z0l1NDOXK)s+oP%o$t)C?y^e5!G9@NBGA-jmqq43hQ{sZyIps^a zQofWc z_n{qAR>XTphGooeRJPS*NX(~QQkIk-A<%7!wbY$y}ThBBdS zC=<$tGNEiJ6Uv4%p=>A<%7!wbY(yr%&AJV1d++@*vK{i4?OliGUj&%dp0`6tqfVF@hR-rj}_dBTPU-FgwC11&3@|FB0U&&wcmHZ`N$zSr7 z{1v`(-Ps%X-J#+5t`*Q<@`!z9hj-=va-%%0#yjFT^^rUz-^f4mjr=3u$UpLp{3GAU zKk|+IBj3nB@{Rl>-^f4mjr=3u$UpLp{3GAUKjGVH;w;ui?WjZlSayypaEw78TWFM5 z)p$aj6@4Oq$QSa5d?A0x7xIUEA%Dmh@`rpOf5;c|hkPM_$QSa5d?A0x7xIUEA%Dmh z@<;fh^r%tZRO110ka|I0u>9l$%TGSA{Nw}6Pd>2x9l$ z%TGSA{Nw}6Pd>2xSzeZ#f4Wn-CGHkOHHW0_bsmWgF!nOHU{ z(}yWTcWsA079@P84Kq*7BlE;OGEdAS^Ta$dPs}6p#5^)j%p>!}JTgzrBlE;OGEdAS z^Ta$dPs}6p#60SDwGlj~y>OhmWLZ_mGEJPhs%#3+kdGXzW}Z1d%{+5VntA3pH1o`{ zXXcsX&CD~$n3-peD>Kg=OJ<%qe#|^`%$RxRI5G3gv0>(!t;4GjB#)pl-W}FlD)yyJ;Q^N2eCV=I;ga199OSf>zjZkK0Fa9xBoZ=IRP;s(<)X9H$nCSV-K z)b0zHW)7Q%!eX2U;m27J7S4nS;u$l5pTRX0@Q0Of&H@4bTX+@-Lac{Vp*jl$_bHwQ zg3z{Rj28@?1rk(ef^_4Ii)Utd76{J-Y1@hW;L|e|XMpHugLHSRH41I9Zgn15i#qoM z=W5_gkQmkzgm6X(&fLfq17~MqSf3DC!*fv2T$;o=9&x1G-8g?^O(w@@qdpu=i?lYJ z$uYU6H{Z6lJ2%sVxys%bPj2;(6t?=06{2fA9pQVCRhfKr&8s*|q|fhOcT3kj(+LALomW+6gd2Yr8U@!XJGu-3#6`sz%OOiwll zU-URMrH(T}_Fne-Qcj{yR%e1_N9*+UzxQ$`SY9v6^F>H2$@4|B<8{;O_U!E_)0(WC zR!?`Wd~L6rRnXfM;+TQ{vCk6-mNt$lUV8Xl^}uY2pJHR9Q$SI#@TYF^_+q zak1^(bltQjJiZ5CK5_Qs1n7+eD~gA6h3~-+o;h!*wL_(&I2+`6Up9gDKt}Ja)@P1j zEg90dCMvQ12+m-1;u*(vQHk|OE8rYk0aotp6{h5k5uANG(KU)#UbyMaU|MQ9U z&sM~9%~WFja|>}XuAd^G4j;}Y)<2(*t^c_^c`9{{oMSRR^{qD|51SdRS5#-2Onx3~ zf*Lx@t^!iEfGY!!5)GtCWOL{X6(DT$WLT`Zdo@#)e zr=AgdgQRz|0eYUgM(7QZ-al)Ao~OPMdc&mmsRrnI>KvgrLVBNUfS#w`5qhJf_hbX~ zJav!I8za3ZX6rl$W$)z=dFr35orA)<$uR!H@*g>p^B+B$^B+5w^B+H+^MCr&Isb_h zIe(#$^B0Rbf2ox7pE{NEpE;BBpFNxNKl^OX|J-vq|MSo1c$SL)=Y9_BzfaBa43%Hj z&QSRj{V`%K<8gI%iZ%C16`FI$;Gc2)@xy8@<0SF=@iJbQrk8t;3UGE?WxK_kcFsNF zw}SXq{d=foj)7*;J%5G#-G+P4bMSgQt~uAi>sxTm`3_!B;hJ+E zyuKCJocG}M4qS8YgV(p=n)4sL-i2$t#0zV2n7AxhOk5T$CN2vW6PE>xiOYh;#AU%^;&RbFFXltd_BH0qo7h|J>__VK^@hEI zlWbczjFayJ>}?n)7inMX?N+vp1uKc0f|bNg!Ajz$U?p)=u#&hbSV`OztR!v=*lDH{YN!)bSB01*_jy2n=O}xBD$0G#WI2O?`wt2=Qgyyw` zmkndvBI6O#2IM$`;2JSbaE%xzxJHZNc2xbsJ1T%;qf*HgQ!3^SuUmu10Uu_4-%*bwbWY>4(GHbi?88=^gl z4bh&&hGL$oKcA=;DJ5ba59i1s8lM0*k&qCJTX(VpmY;z#J)zNYq#JM#$Nc^}ki zqU&7K*s!j9*1QPKYsr2Z*7Zf&tWsC>gSVpR)Opc!>b&SVbzbzGIxl)oofkc)&WoN? z=S9z{^P=a}dC_y~yy!V~Ui6$gFM3X$7d@xWkFq}oA0fZ?F4qmBGeP8m>vvaT&92bA zmUO#eJzS){rekg&_$#_dy%b%fUWzVKFGUxrm!gZ*OVLH@rRXB{Qgo4eDY{6#6kViV ziY`(wMHi`;qKni^(M9T|-hX)ZUB+6`ooh)S8`hmg+A4x!y^x3K4RuKLhB_pALmd*m zp$>`OP=`css6(PR)FIIu>X7ISbx8DvIwX2S9TL5v4vF4SheU6vLkEdr@MrTWwsYan z=J@*z%NmyRBK5S65k26c$eMB&SyS#JYsy_@O}UG#DR+@I= za|*tQc+S*KRqRgJGh(ZhiDi{Cv8+-imQ~8cvPzj)Rw)z9DrI6>rA#cVl!;}PGO?^u zCYDvo#Ij16SXL<$%c|E!cRj!p<(ze8PJQ!@@Z<2J4<52SJhy7!2JxA+ z7XBdKy7;k;&V3rblgj5^?d5q_LDDvNz4}JDE!?y4e#g9gqS z^L1HFv+)eyV`#P8W29^IIpil0vnTmHuZxlBBxx{B-uJbbc;5LYpRrrxlafE4t)=qU zV&^1}R%BeGDc#4}aMb;ftNW2^x*t)x zAKVHaySg8YrIhX~U4wr~n|X#PL)00Bc3T{J<>;Xu%jA>}byBaPOYkqov(y+;_LNJ| zx7K1>7|-xMhQZFr80m)j9C_Uyflfso-H#_H)p=}4llP&cc)qOr;aE!PB=A7loXTHY zBqw_dNJjBf>3+u1{gzmi z>Q&bLw4>*Yy8t^-y06&-FcI=64=Z&)#4`y)_31v(8&tZF^A*8A=sx6UMyErf`}h_d z%R%=UPu+L)!)U|Dl#b`EZe2ReFNeE?)$dl zcxIoWaNie87K~U1??Ts5ZsMSUQ3B@EhcN6*L9dX;Yd_8fLpuH65oaUt+ZRR}LmM_p zSsK0#7Smc>A9)OMK%K(Z7Cc8C^K}}N&hx;oRCtRAr7`1qyIsbN9Jpd;p9cV zhrIQq-F)i#I&&%L+OU(aw=M?Tf$r(Qp0*?4Hwd<{j>L5+ma&4Ya})ciGejPsitO z=ySqBoS(=04LG-O0MDoKO`elkJUm=@8spOpg+Cp}vkT`rrZIk!-2~q?ckmp$y_b28 z9p4|o`y6{>dRV2WRC*Zv^QH$?dPJr3oIr27rP6Jc4rRI`J)_b`RXWd<^u8Zg>3u4l z=Pi2EBPuOK^o4dq;4@SsM79JX``c+X_iVmsM5y9D$_D5EvM4P z$1Bs~Ds5V&O-xj#MO4~hl{Pt9nP#iBqbhCNw#u|#m3CaEZQov*)~(V`sI*&dsZ2|% zw4zFznsWTkv)oa{lMkgkPpPz9Z!M?wFQlDSX*+h5)A|OVJ-BwQPLD~%%m%O2k zXG}s4yUJ-Pq!oS7s&gj!{UO|QY?s#~xaQa{uSapsv0Yw|;hJN+ydK9j$98!=foqQK z@_G{29NXpfHe7RTm)F~I&9Plx--2t7?ecmG*BsmB^{u$(*eqyCA`QU65esDGk!)IZTV>YwNw^-pw;`X@R^{S%#|{)x^}|3v4g zf4w|Y3FlybzKMIl?y`=F9ue~#Ju2g#q%B9sD)h*Mdpdp*>qM8RSE5VQE72wDmFN=n zN_2^OCAvht5?!KRi7rvEM3<;nqD#~((Ix7Y=o0lxbcuQ;xrmgoiR zP)BE0R-Y<###4_9_ORX*eWC7%zEF2WU#L5xFVr2;7wV4a3w1~Ig}Nj9LfsL4q3(#j zPW=6Ob?0x1J=kk~nr$!awLbMe#URlOw!0j?D7U>NZ8T!(^r3!KUA;K`@2VLTBTvrX>EzuYdDwB_hVh5XSj7n-b}1Q(*X7THq| zME2AJkv;W5WKTU1*;5Zh_S6HBJ@r6jPdyOXQx8P;)B}+{^+04#JrLPb4@CCVga4Cf zZ($GPlRxHoAbgG^&yHNneKAQ}j?63MioTf8?8f(WyQZ8)wv@BTmU0%^QqCe<%2{Mf zIg4y5XOS)CEV8AXMYfc)$d+;z*;39TTgq8vOF4^dDQCTn!1KA^>wb;!G=sc3GA#GC zC2cwKt&k!5+Cr0fFSbc}iu@=`ksoC#@}n$8ew3xikFpf`QI;Y<%2MP}KJL+SNlC~TfRPY&NjY5<7r|UlXF1#k+h1cY}@S1!VUX$;_Yw}%q zO}-1S$#>y3`7XRB--XxYyYQNP7haR^!fWzfcul_R_08@&bzaK99DY~u6l1$WlNcqs zOa2N!$y?zkc`N)RZ-t-at?-k)6@HSp!cX#6_(|RhKgnC+CwVLUByWYEoWnx*SOf0LEiDi{Cv8+-imQ~8c zvPzj)Rw)z9DrI6>rA#cVl!;|^um^b${^%K=gH0Lf^N`;8T*{L3lJXqqBPmP1!?{P6 zo7bFYWVw0GIYpM6*PK6Oxp~dGLYAA?oEK!ddCfUMmYdg{?_;@n&AC06o7bGjW4U?F zIXjk{*PNeYxp~dGIF_5&oOffndCfUi&ao>QYG?j3zZTZ2CwH51p8dND{NQ=N!!3q$ zJ~G$!;C@p}pJijdp4V8AfO-5>A&hHZ>!20J+LpMPDkRKwA&j|OGq6FeXAioeb2c#7 zwHv{9qjZkv16xzCzF|g^STcccC2TBF!#a_6u2OjWqUj5p^Ef{fE6t1_;CbfTh#$Rd zSVpn(oZ#lVWyKrvr}=C0;JVWR;>COS$2G2##eHw$Sl8gv&;PZt7Vm{iB}|bMn}az2 zH-xih?JDR0-jj09{{{WMJpUJAl;{5A)(LxY<}K?3W9r14 zM)WCz^}*=W(Kn6YQwGlq=6hDjz&iA2W_aeVoHfY&#Q6@M^Ffz_kw_sBi5HAW5^Hy{ zMs_Fixiem9*_pz9vM?VBt8;_r;}IuJ#Z?%Hg$p*;r3YgX)CV{}7|kKB>u$lCIIicz z`ciequZ{J1fsRzp?h0cqIM$|O4Kdf<+7YgKJ$)&F1#MiD7{nSlD-q9QEpBcmoX#0> z=X_scUZ4|8Bjf35yEAS`8uK2&y;>WNZ*iTu4O+}Y80$2x)^y(Pisxri-MLaan+<|q z5Hu12qYz$&6;q^@O6M5gN4_PW0>_JU;m#BUjWxGwUFqpdK`g@Kd*QZ7-tJ5r;MH^h zYgz}V4!>!%K4tI>;b0=N5Fe+E&6iai%ZK%I>ipngJ`bLAy>BaM$~nTUOM|`-sdI#R z9aig;PhZ*#0jhI^Q_*ZGiFG~ZmkPU~M|(g&if^iYm@oh7NvBTu(d$8+=bSf+XAJB) zFyi-K4)v7x-tpXDhc?dr1wEddToZqe>h`J!G>XAlSY z{6XBeYWMfy{%-A_X9eH=%2)Dw+&&fewTd{N6@2$AzgZE-vx4t_Qh;p`QKDLY#Y6aDezms7vat1Mc?J zC+sKlJ8}0rNqoopoiD3%KkKPm?)KCzLO12oO_T0#H$c}@&j{U&OShYJzuf>`Pn{!l z`&_#Hr2EYV=z8iOp__H-4v_9Q8ldZ`i-hi=OLvHLf2#qyo_a~>4!d+mNcT4ypzEol zgzl(IcZ_tu+5lZoeI<0qUAhyb+t~nJPu(SSCtbSRNVlT_x}JJW=x%rE-a@+V4bb(} zX+n3(rF$#sCK{mYso#X|4wvq2q#J91uBWaOy1QJux07yL19Uz0p3uF+rF$powtl~E zJo{Ns9f;jU#bVJ{0?ZeEzAHo4kA|@s+h4*telR1AXmGh_5IsaF_ zlJkG>-E)jtCxpLX z&HbQoBE5-CNoI!D%0AeLlY#cfXm$H%oR{^-t=d9SHi0?AK1Z{aT(K z_Fw0pnPb1lwCx27P3?ukme zw43cN&09dzwBxx_JXyDWwpdrvopR~kO1g23(R9CDH{IR#m2`KwbZ;ZwZqohDy6Jv& z_my;axpZ#_-8Rf$v;}Nz)IEEy9-M01cO~6BT)KCHZV~4wljrxkaUVzA!y7aX}cCl+qXzsZjm&OC-FP?x!<`T-;ua!H||j_?ywtoM2kD>#vRk*j=OQ6 z*5XdMaRn{Ta^r$pobASiwYc4G+}&E-J~!?j#Bo2*p_bCzXRXrQuW0AEd=Y(6)_1J; zSm(2>?^vA&yzuyI%C=OoyX>+J+SOL z&fA`8)^Qc>t(C6hXe)aCMmaXrR(zk;aShi+SE=I!^Y!}aov4fCJJsqN9na-Cy>67a zAJ=;6Z`?DjXVA~8zs`9T{j6`Pe%3zpvp%NxvtC`fepWrS8|`N`@{PsnvubrsUEipq zuBnTz$EFsmYwDuAx3b+RbgT8J>Y`gmf2uCJb@Zp|qU-5Tu};wI8NE)Zi>{|Xb!A;s z?)TI~w_3kvyW2)p{VCP&`Ht$J)I;~4D*Yb4Kc&|R_0VmrQa9@TDe0f+eIdbMy`GRb z9jo2CW1sY6bZmF~F;&xqR<-(}S|4Dq%U|}73P(yu? z2I~Vac3frLtsZ@OSL%Zr?d%`hI>2K?g4Naf8+GxzM%{9))B(iQQP&b(UTS2=?jtEpbc5Zyie)Yx@gb9q%Qs8gA>jD9=^H z^5x+6QZf3tD`SpJ{kePo_&)di@t%IcA-1$@`H0AI=0qD+tY3r+LgFn#@Bjid)vZ7yQ2NB+QYx6+QaqG z{A1)}C5+Q?nR`nZx6hS7$G*A8gmF6da<2*FbiCu96UOOy%)KX!)3KN9jTv{?{RYSS zx%QZGI?i$pGUIIbIgZy$d;h2#$2kG+#rjEYO#UYtY7aMYR&?>l8fQKBv!Xcb!EoV+ zji8%)sAHqxppK1#gE}?}4(iw_IH+Tz z;Gm9;f`d9X3J&VnC^)ENl;EI_mx6;jHr9)SI)-lN_%M8}3^v=D&VGhP+Q)%wwVheR zX7n~@!x-hk*6R4TvKTe&*5j(3xxyHw<6pxVRRgzm{L|ZOI(7+8>DVPWrDK=il#X43 zQ#y7DPU+YsIHhBk;FOMCf>Szn2~O$QB{-$yir|!vXZ7Ngj&b_fWejb#`qvDeN1yly zs!x0$`ostHKJodL>l4>QyWT!=U9{`Os=8=<#zV#C`&_$Ze^|$px@gz1ol2}K<4HZV zYxVys+G+|Y;sqjsBmGRXU#3L%Xd8e(P8zZ5|zC1WR;$5iHU1MX*H27r_!8 zUj$2Zd=V_s@kOvi#}~m89bW`XbbJvkx!2Vtu0Inj(ea>OEYY#)z5vf;w&vc350fx_ zSp82Gnsfh%waZOh!C485;Gwq-B(|S%<<-3$+g}o{tgP)<;K~ZyzK$ym+kOo^)Nw^` zOX!#&_OD}t*uRbmV*ffOi2duBAoj0gg4n-~31a^`CW!s(m>~AAV}jVfjtOG_IwsU> z|2lTiAKE9((%fGwUp&ry=zoSklKR)j{P>Umh<&D9`{WqAUN_e>4!JbjFWYB5v}@JP z6>W+YZtLr|uiIxm`EK*f4+y_?+n2gow^^}G-Co5ub$b=t)a_MlQ@2;KP2FC_Hg$Uy z+tlq_)dGu^Zi%#BOw361&lDN$f_qC7I8#TwUaz1ewpU-8jy1=>6kwH&mB3 zxVC3^<+?!8?^?D~*LssB*-p8>sfTv0`X=M@lUFSu&b$u7z z*7aRSLH$@M1-4s34b#tkD$T{O@ zv_8Y>CF5*d+q>MD?+e?rf3X_QQdi)i6?+0pb@^7(b;ZPfWudL>q2QISKcXkP?uee~ zx+8j`>yGG&t~;V9y6%Xc=(;0%qU(<6iLN`MC%W#4p6I$GdZO!&=!vd7OVyKq3ty(; z&xhvrUInLg`-{6a&$g!URp-C(Rp-C(Rp-C(Rp-C(Rp-C(Rp-C(Rp-C(Rp-C(Rp-C( zRp-C(Rp-C(Rp-C(Rp-C7adiIcZJftod(_ifer8IUh-PycjQw|G?7ws_g0cTLee6F| zd+a}GmfJRB7hZoev1SaknMz8kwd{4V;mdN${|kTjo4kbp}W}xbujk87>sRfgCK>;*d>JBrcKJGElaao z+P|n&DQq`NNt8e{*t0eCC-FuJc%wU{NSfpl~C&7L5 zopXP@d%pATeb4uut7Bi#e#CU#3fhmD?_ya5h-JZbmhw!Ow%`9Is8?Ef;+T)vu*s`_ znlWONZ}{muVBfZ$9sKS%Z0Xs~G_}^Ib*#_qe~!J=V~kyg_^(;3o1l#Q<Dl(ibi7zo{z1eaM186H zH39e9M!kryhuF+@2u2w7ir9i3`^c9NSN7Q3F~s4hj4O+X96!^6{4C?Mf1mZiXB_x{ z$AMYTpIyuQM2$bo7;Yt`xS1lJT~K@iF}5Rfk&|pX-e=GE!h)*Hb2Jdn4 z$m!!X;9U@pd47H1dDx%ck3WBWpFjOn(f{!4?;_3((xuWwXY)Aghel%N*L{}HXnl-a+C?|~G zHQR%*jIa@x!Ir~Tl~C`i&@ShEiJ;Y9( z#UA9TWKf*LHKU|2l~M?YRGOMD#e&2B{Q{(#`Acl;6ol_9EdYTTv*0iWg0|cD1+B} z8^#L6r-cslG4+=P?JSouYuTPiUC>2+o|lrTE|x)=am^7$CUMlzyrkKWj(Q)+Q7dpz#jFst)^Wo0^1hoo3>?A zrk$1TLjCntZrjCJ?%1?mmCG@@2IDvQgzI2@cSNb;buUxbol4=FY<~qDV*}qe`>*pn zwleT)(;c4o>Xj7woq21PHf&C%!%mL%J_cUXzJ|n>RA~2&2XnE%{h!yA63-bvUk)=K z_ha{KYW-5?tij?-60jTf7&Q0$S4s`&m)g*jPyBLw+R};$u&F_P-xTo_Jt5gMhq-Q~n*OA|Bf}6@-OU zEHRvl+4f^UdIZD|Jd6L~@sj293^By@fd3_n5%h5>S;_r)G6eS%miuwX{X}>;xIeVq z4@1lSFk!eK?M@VAJbq(*v@j}(4)yEfl0Q+K zfxfMiI`JjoAb+EMjPGpPR~r_jjflfsSjzIj$rQ^%NUcEP63hi$uXudBm6Rh63neZJ`ki?r^~*QMGs#YK4;St)Pi=`KPBeLnJrOMT zQ;z$|Cipwc2ltyS_iO7?u$$uka`em6q$!{HW%z>OK2ny6@X%MIb?ig$YJmHOaG!iD z#iyr&2h02-_k+5yNSpYGNiK1?gC(Ki41I^PfMd&&#J;I8S%@c6!|{Y|KVp4*!P3yP zIJs~6K}@*M*x{P{-+~`zk|}T>vBi1c&p7UYjEZc>Vs=YX?xlwthqa&ED0Uv zV;bI`Huu*g^k~Q06=`s)!E!&dyo+T}Ci&y_Dd^iesh?cdWH{-eee^lgzWTHokJ!d= z*~&s4>jWp$EDI_1;;b9Lnew{gz+;i+rk#xc9p0I4Pca|lrd_|3wrz30ORJWfc2>3v zvDLwca=Wfw4Q?a%%jJ^$QD8}?;C*;os-0Y_yW?Gxb^0l5%qCC5JTSqD{vg9KXH7I-HkZ%i2Dv64`;-e>ylpO@Bd+a zPsNhx?>Y=8k+2xxJX)6y5Y(~gFMY2gemOqHx+6<|yqPkLF&IlR|HFCYjxg{!*Qm8~ zv_w39x+d;#e%#HsJm?&0ycVr-`bQh7kGUT3V=KIXcsv%H%AgLGcYH+{EI}F6iN#}6 zCGV+AvBt#dX)kC-554Bq+~y^wQ@>Ze|IM@)o0l_wypKIEnMSY=pU!)o9`-6Tde|pw zQN6WJujM23um_JRFZ&UCtDIibN9bjoUh>9zJ+5D-y|8(@~ z==`Nek*^2+*?2CPDUDqy{sF!lW$JrRUy69Yd+Cvb?-v`rZ(M4;(m2KUWktM`+46zk z^a#>l`!2q>e35%p?|gd;`kHNc6lIBCm($x!y%P7V-T8JGY(YKNDS91FuhZxij9v$9 zQmxnK^xBQyE~D25n^o(rb$YEv?-8T77PhO_YjJvOjNW#m*8oq&QyNq6+(c?Z# z)p}W{x7z6S7`-ewP_4Jh>3z=Vbs4=?;6k-tlha#i^xBMG6Sz{XmvMT_jb4k<%YZZ0 zdP%33GI~u$F9}Xn>!qDugVD+c??!0*LV)2oK zub3ElJIULZ{ifd}Z~vCOEj3r)KgiFXo`{KeXZ9rNar=wBSSm1om&?DI`AgI0I_||j z>U_pE`;lzL-WVmT+u?LO7tqZzZmH63bGq#d=(bhqu64StT6cS-B=xp9`5GbskrxV)63b3HSqd`NH~rF^Mm{`j*CKukUf-YDu>`Uzz4;}n3gJ~b_jl5^DTj`^}&plY5_h(+k z7|HRG{>Hf&+xJPv$k>a|UOzrQd&=B^Apio%vVHbF@F)A#e7C|7W5l%jocQUYTw~lwF|x_ zay^H%K2e@Wiq2wwBRY$XOQMr>eaty!g@0g9DLQGV!+9mwW6&={hjA{+^%%so-*7z! zG3_^8k3p=*<>lOm>oJIFAL4opV%kr*9)ppn7&i?}4zl9Smj-zYH zQ9K(=t7iexxgn1gw{FN|#dXc&3XbCanAD}`HTu4*xhp)?+!dZ`?g~#ecZH{#yTVh= zUE!(buJBZIS9q$qD?HWQ6`pGD3Qsk6g{PW-!c)y>;i=}Xo`Zk;Q!)n^yRUUTY$Xrz z3_Go!Sw-ijxT-j&xmv+P#b?b`xmRj#3I{bWg@c-x!a>bT;h^TFa8UD7IH-9k9MrrN z4r*Qs2Q@E+gPNDZLCs6ypys7;Q1eYVsJZ(62=0}L{cx6ZUc`R*mP!4YGyezfox-Cw z$2Gn?;roU*B%pzuv|Q23@fD16f#6uxN=3g0vbg>RaJ!Z*!9 z;hW~5@J(}2_@+51eA65hzG)5$-!un>Z<>RQHSkTunS4ggH-G0A-}{6k9gatQN5^*( z)9O8)=-djf?ZDq=cDTRI@SUC1)$H2C`KH{nvJT^WI^mG!m2gP&N;sr>B^=Vc5)Nrz z35PVVghQHF!XeEo;gIH)a7gn?IHY+c9MZfJ4ryKqhcvHz&hPL|{E3@*-zzrga{bMB zR(y7uR`0Du=T`7$J2ywF?`UTCfeFEm$#7n&== z3(Xbbh31OzLUToUp}8Ww&|DE-Xs!q^G*^Tdnk&K!%@sYLI(ifL6tTsv;KnxCeVem8 z-~UTpYn|OWpOSll=7QK<^FVB_c_22|JP?~}9*E5~55(r02V!&01F^a0f!JL0Ky0pg zAU4-L5Swcrh|M(*#O9g@XL&Ecx6qTEx8PgoiJO?Wh%G*@-TPqEK4(+@%}2^XWzv}*%e%1Xi{i^$4`c?P8^sDZF=~vzV(yzMz zrC)XbOTX&=mwwg#Fa4_fU;0(|za7)iXR&v_o`VmNrwgrFTxjhV*D#8H*9UfuBiAmH z*DRyMVO*!k{wLPvFs?~t{}Ws5Fs?sj{}XF*7}pxI|A{p_jOz;7|HQHm;~GNtKQX;t zko`}r$?0)zAp4(K#$j9s$o?mmbQstCvHyvs9me&2?0;gR!?>1@{ZA}#7}xEw|A~1H z;~G8oKd~(i<9bm34S1f<_|D4rwq99=mCw}m@NI6xML*4NbsJv6qI& z^DX=rEc2!>jd`s1!t3~EAmcMlCNieJ`fE{sFK}!b6mf40mLz=rZ7#2a-`t`cte2g_ zH-q>xjsM{YCv1xK-R9u8bUg4qgEz<8!Hg0I|9~|uJ?%tT+*tKU<|He&w?)J6~jqJ~lZ_ctQ+qU((24q{-508%Q&5!5C z`X>t`{Ri^7iP79-p?@+rK6YRtH_$(ln>aK!Hab2Dje-6Bd(kAw^SQzN?n8TXLnC`e zb9+X|bN$1^xzULnnzv_YI6tv|Vtk;Y!RE)uXl`{6R{S=0@N2!kKlLkceyOz@Uo&|B zYGHn@#Uq1%2LEI8lN#n&gZ~>M`1PnQr}rL<-zPY}&Ej$8%O>%@hQ#Rq(EfgopTyue zh<^ru-u(1t9ZyaQ{$~~soc?zNf5GBCjzHfN{6&ku;)waG;QwjyGx*qy$J_W*U#g9{4?!&!y-p`p*R}x)8J2A{Cuy)e?{!l z*UmA)@eL=BRqnc;6dWIa@_63qe_Qa^4eT9r*Y%gcd0j78UDw}>{ytmIc77j+GX1dJ;%nV?tr7g6Y<}tIt%CoH#jm+?zAE@CTh6eH4R z^0N-_slxGroo#;={;rrtNow&K6$8r*g=E&;rwKNa2~yHXy5+a!w=^kd@#3tbYk+shks*3XYMOs z?eIjfFj`jcT1dMREGu^`q+AJ>mD?9mwqPz-7#hnB92y$Tb$0CDknd{Wu&J%TeV}ts z_k6?41(_>qpBLyWDx+m}HolBI?Mkq$%*J1*TnUzy+4$>}EvT-jYt!z5_IviUcXw{M zC*RhwX};m*g3J|dm>1|PDx+m}HolCjD=LF!Wj4NyJLO8Stjxw=r))uWMID_R208}r z+0-`JJvh+S*;(CiLzmY5gY$T~0#@9iMXEs+csUEJzk$eDae>lqgd5vv{c9(aiH*IbH5V(!Ut1tN_DA#lgHf z{}0ElSkmSSZ9Yd;9xL&aQ#Pqzo|`01N3A}etJqeVU(55XMt{OhR z5R?2D-E-4Lm%lIN6kSqPy|)W^MSeGbmHbk!lv15P4*{_@QE;i`-&mDFeo8t5ed&8$ z#Cy{gZE^fV$N-}!*Dw4#47|FHET`#|&3}qN)FJf>RKm$Z`Omrh%Uu1UU!DJd0nJKI AWdHyG literal 0 HcmV?d00001 diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack-.hsaco b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/library/Kernels.so-000-gfx950-xnack-.hsaco new file mode 100644 index 0000000000000000000000000000000000000000..1ddee53dc815a9e893857345072b6cedccbbb6e3 GIT binary patch literal 168368 zcmeFa3v?XUc_w(fy1Kj2X!Hw>Mgs&v5NuKq2tptPK@f-0ATlk|jv|vX?a*4q6fII} zwIPY3D4F(9twx)qC|Q0n8O8A`j$%29orf(y;wVm*Aic>>CW$8^m7Gno+F8yfv+Lu` zW=>`^=j@r25%&9T)sMbiY=GznB$;A24|uEYt@`WU|MA`Tf1i5y9k*M7z_hL0Ur=3A z0q>f@w0HgTr!9T;zA)paR2%+Js<^_f;=9v2_44rl^N304c+W9fzUf8P1m#oit%UfP z{_(vKub$SoO8r7@xc$GmAC%C1k9nP*b;~jRE4ATakIcEacVT>w@4f83_wqO0_ayY5 zRzbP&-s4lt`JQ}N_;S~OJbF;`bD8<{*sJ#7J$|pzAHT)FDYsacg|B)Ulu%?A!8`vr zq|^YypZw_^J9mFTsZW0Wz(WTMj~y8O=%FJIADVgW7yrFV|K{j{2M!ga(Z4?OmzPyYJoBlq9`*dazd_TVQD?fT8pCl1U!erW$kAAI=WS3mOc{rYab}#ddPb|E#*L}k>bM9!_Dj{w{n_a{rFmI$cjUo?`wu?&&||+h z>dErZ0}mZ~_~`!oA3S#Gpq8xqu2yp>^!~@Z!~-)AKJc*fJD0^%5q=QZ^|jHN0}ns& z_<;uwef52Z4?X;@I|knM&4sA_PaHb(*n^Kee8y6*j6QVW*#7%x4jkS8u}6-4w5m)$ zuwgI{A9(1{ci#12@uU0S|Ni~EA9?KP&i8*{YX1j+X5#lp4~D@f zwqS+9e!|T#{AARiO4}Ij3ncuBPdGw1gP@Wi3}k30gFk~`D&k>Yig zI`Q?<0|yTtIrP|L`yV@e;J!n@_JCF#AA;1V*l)X!?ElE)_uqf$$Q1}UbyTMz0?8{9 zSb&V2`)Cc2T@H5xb)H%p?()lgqiQij%Y371E+g(6%ZIzs<-JhN)7bJ}sAg%r&I?Py z-Ndqtyi?7WS1&B%ooe3hty7ja6YK_759}5oY3EMC zyNQPIu5Ssv`<~#PkNCH1*%WqaRf)twa%GwlY#ppS<*0fry8`bJYaW#ptil8;EcEvRWi%hF+eY3hz94z0u zGRTWH4|%P|QLAy(Y8w`%La=^8b_;u6urwqir@BDD;~ufu0u!vdH}Z%-1qPn z1pLD~#w}YOxc}HKH;rvO_AnMTZIRWDvQjI+C6inS`Mv9#S_j#hz-vw5=|!y7)pYw? z6L_uB?8a7eHZht#v9e~^rsaTZ{RN$^W>~8k)@p{WW;3j@+5SeFVXH9JzqWa#Xf=*n zjiXlMNbi!`U!A;bHICRhT>hARgN-9id}WQJ6|8P*HI7=1qgLZ+H5*5bfkh*Yqg6nP z)&!nx;%iOdwI=Xd6L{Dc+|X);Ium$}Em&xvFLCOTHdvjsds!#kKau)pH89NPlqoc-KUKu(hD`s4+X^tvasp?^Mq@LAB;@ zIg!&o|5}Zs9|I?H9N$?!STxc&(v(-$je5%#baFlCQ*d%QfN(LtmY;7eZZh^-w6^ND zw(72uU(4SSZ)~1l%dL|QVbb~~9KF3#m-Na&X=*t=wpCbVPE6Vbr^iiTQft_}HEiA* zHowM)%@?q*I&9t`vi2{5tiLbF>ZiUkaJ~Mm0oPW0uQfj38lOM%@Y~|Yyl#7M`KvqA zOQ6yM=9x{&@s&Z+-ER#f={<$5_8wORwN{3;R))1!hBdk}YytaxEwbe!>rztzi?#Pw z2(IsaYrvK5y>e@O@6eSTr&m`Rw8r;Z<9l!B_}-ES*Bmzwu4%Zvx5ACie^$O6lO``a zx@P4Ae8i;pw8r;Z?Y)N&wpM<)V|CT)r{&7)imtpCYwsfA%W`ZuR%swx=^##Ld=3}* zeRYTX*hz^GyYXLXcjMoWkEMbUb?$_FKkWwUzsvJ!_xXSJ4vd^Z$mh$I+UMVeTm4wk13r}!8^2HZ@L%2d*FFCHlElB}#vk|Szvz{>T&|___g?(8j}Nza`BA?7 zbm4hjuB@B?5q(IhQm=S)&iUwH^yrj7S^M3PlhFHpjgNVJ{!@wn zM>k&h{4U|c58U`;UOvBt_>^i_-}KHmbxWbt3GchpK0g0Dp;L69+_%=Hx85t)xmovq z+6~kRiTCmCM-uPj!=rdlm+J+Wz&S6U|19x;>3;Xvu$z9H@c(~v?|%ro{Qa&+f7(a? zMXy|CU%CE4%Io9vVJUCG{cazgg4oq9R12qe>(c%O^CY{akt_A z8>dg7{KQZ61rCqLKK$yXRN|Ju{iC1Tf%xsaM!xd!xnKGAe|zpdGwWlYeDPNA{+~{E zChk7{Z+`CodF1JnN8ca+PT%-X&3wJ5{lle8yVt(!n&72SCtf^x*VUt!~gh4-@d!&y))%6eDcN5|JqAWJc|1>i2tWU z*){*i%$~KX5c;(vh#N$Hw;{jN&p#OZ^G|>IiAj`;f9vs=z+XT9`18z9^|`k-{}14< z%Zu5JkUU%G-IHhL&od|0gUC_*W${P8Z@?daH@Jc3uXm{f&pg!{+}()33H*)YZzKLF zc*=#pt!@BabY*s(cYhN?>e^C2YK^DP`Z4OkVm~>?S<(rMo<2#G)@!+;F3FE=1^#G* zZTMsVj6d20f7E?eD|!4;ALZ9A7e)xzfLb5b{{8nIJFx%o<43T|^T8uWA3reTMIAge zbLc1zm6g~K zy#Kz(ac=J9#7DLsni}7F)7XLW`zG(dWg#n7LVEgnd?CRsQ8l*uI5S_3_nxketUj!_ z=KD$4MphqY<}ah{Mj9oWy6GeLjo*C#_$`xLZ$30Oanl0zDj|15eIdat(bk37>f_9O zHQp#uHM06JGhdDO9wOdV-F(y7!CMaAH#IrAkVPXaY~U}% zS3kfH*CZJks)?_@!p|>~q@`$n$jAUklovAK#cSD^@zt+rj+jYC#<);@fmHPse!V7R z##d_MtFQ1Y^)hx@eD#$j@~rj1%$rgCMe?j=S?86-;$6)&U#RBn@-*-Ju7NJ`Qau*v zmlk1LHO&A=<8l|l9ltUwkVSsP-`mwU)Cq*dX7e}Ty+6&0agiVEsP8ZGzI=Q$9iXYu!0*S+gX<=?yb z`>Z$5FY>;8zr1hp_m}hi=>vb>d45r4vg zrSa$7-!1+=+pB+c1^p{~<4N_Xur2x!0C{|Ih2ZkJsPEUwIC-PV2A5gO(0llREI+oTbh{_SfdGY<;9O zR=(%;tr+H+F7IX-Kc}p7x)~R+&K(9@wg$tcr_Mb$7n~2CnxDt`N6bTKbh++jAfm?*t5mLoX4q)o!9gJ)Wz(Lmg>BIQz3ifP@x>p7nwGO zv^1VgZ?!P)g?Bgb{?x_XMoW!t87byA<_oh!V}8*K{+A^qS^CLPf7z%@Kc`l8(2;NWOedaHPGG-EmBIDBFc`6YvM%E7E?}aA=cs{Zr zu4?aR^!pP%coy%(Zm=MflNUQSBkZsXaa>QXv6QvJE(F#5g~)+z_7UWX^phdPTe(uu z>Md2mv3w=omyd;y6X@W&W8tC-BBul&L^EZhr z%F7z5rSmcB`1H44O9lc?7-1UW6JDHky_I8Jc#D;LT;jHRaW`4HqY`(E7xxoZPTLIL zv3}Bvd$*ON&hi`9d%U=xwsIP$7`MlZ`+$|xc*VF6S-BX(@XuPg^1&0w!_(h<%?Vja zhhdME>{3t9l`XrJ1dajOteE=Yg^4!HvJpXT=QC*N0<Z zSd}qJyW)E59I#%G%>(Oq0qe^MIX_T7Y%hZOu|hdeIU6pA=d4FnsaKtxQ-Sa~yrO1f zR-tS~OJOTqBA={?Q`0D-t}TG0KjFm6bvd6_XlP zRV|u0h=fOm@WF?Zw_sNS2A`b)4@omg=oV*x77!Jh> zZA{~JBvY`E?sQtfZ-kh14kL}BjXK&bYmd#e5T4W zq-*+}M!Dm0l!?00W7B>!wq=Eau@axpgnKcOnZy*)`uTmGvm9GWL(Vy4%=benQ^|NhOIrBxg9)@CJ3<=M%kNhXEmeb%%8++2I zqY-~T&TswT)iOfX+nDRR5Nsz8)di~!;fC_rK2P7unaZrSv#MWg3t0)fM44DXAM)Rd z;GKxAbw(k}I*k)jk#KR=b_$trbS4va^B-)7{Q;)j_~)N2yZSuGxRRp%#Y#cQt{kh? z{q~px-H*Du-|p#t)Oo^wx8v%*~62Wbq$uuJmV4y?DC7Qr)okD=Y|jFWBz&ymle zPG^eGYxyP9Q>4K(c|Y2g!Sn7nwOrbvgSu&_VjpNw%99@;T;{ z?=j@}qlga&(>k9{AZ6QYo$(moLpr|`TI0BR@$SpHw9aR3z93W2h9f#(z`BHGnRKJj z-(VN)p^N3lHPSWxZlm1kG|J@ii~8Q~SXLyMEb;mF=%8(NWVG(LgYSGdn96Qz&%`D* zhND7aTsa%x-X3#x2jgn4?#DgdkL6(Rqz~QCxw_w$h;uHrs{2{+rGvAwB`nCCbEdT>jgz8@t`nIz!uGnvDrs zt36trbz+6~XndwU>gGR`h$_3oQEohS-_;Msxw@Y$g&@0fva0*P2s=#1V$l7#tNTe$ z_v5iA!tai`x*vqK9H_!&c+oH!dp8$XIPh7N3!ybcGhx0%+4XZRk&@S2#$*EaUUz|%-~Y>Llo`DLZ0x0aGTE84=>oAmSxgqdo0ugd+1@g zagB6Me~nS@Y!+p5`9*zC#w@GN!~JBu*S1qG?k7Fm4|T-0C6nn%>LlvQ?a6e;y^d#b z?dlJ-xK{sjIVE}@>5g-)KvnN!u6{G_JZwPgJVMfE9T0}jw{^w9KXrlirM3Rd*{~Wt zhX;Wl!{-JNmeOyW?)CJqoc!0b;hj}|qb-CJu@Ys}g>O+Fq=`7KL$oEzCcGXU{*G8N z8H>*(#f}p(<-adOzwy1NoizB}jVdDOlx$pwx|6mI9JRo&_Vewtz%NN+k3@E$IFdDVXm3&{G z*YUVrPNQ9&?gBvIKApD9mb&fL2)rEl6^tWC2{-7h^lUz<})eQr$O=V$5r}m_vMaEAA0;45$m~T{mLvYwJjX&vpZkyO7qyaiOha#h%T>h1t>Z zLT(MmzlJdWwW-j3(Jv~IX&)0`WzSQ zz<5@qhvQ#vJlkkVjPqG*8NW-vo5wgG`z`C&vyD7|A@=V6h4HTRDvx(HJuc)M?`oQk zZ@i0R4NZ;b_{O_r{KXv4Sr~sY#&Z_NUySjbh4B}R=jibljOUc67aM=MU5~$DJ)$0e zS?`U%${%S<1cx%zu(I7m$w@*iFK{59ic4?jz$sGWNr9Add4aHU`9T zpQtwm#PJ_J_LEVUjs0*u2i^B_6n&X!29s#s*pE5B!+wWgiw>AIUn*Wux53Tv%*8Fd4{uk?hTJyiH`CnQ8a81qsIuo25 z=A3UE=6r+g(QPvC%Xwkl=1$<{w_=C$QQIz5 z?4%C$T(IrV1z$D~Y-jS7z8vO!_g)HGHs*o%UUD2aXINdsnljd|V*Yqx%_`=QWv&G4 zDE7e+(tG5xz2YciE1^T>MsI2p|-ey2y-dQM*N%YEa5 z3i{`dk6wZR%UA3CG1rxH{ue9qEbf0mm}H-5ufi|j?tch?x9k^OzW+gCO@*t&lpWWN zg|OC$vg5Ub`NVL_-T!c#xBmh3Qlam#Q1bi)*eQ!u;=K zq31fx|3)VY=Lg0L-8YY_p6e!yJvWbI{?|>Lw+6As$ld?I^iJl(qt#7X*Qp}oGFYS4 zech(w?9h0jv;Bo9XW>$Vw&7s!L*~XI)!9B??7VT4lJquj|3e0GT+`5r_p``fC+D== zhl-5Lg6ExB;}u%J34iYXhtPFVReL|8-_KyqnE8lfZk+SN3CxAZv9^zEnX>D!e(btU z1qV133f!jWgSnos3+1q|E+dF_8I@!@UvUQVaqNFcVE;oB`yW!=|Ilabe;6?LKkU8K zX4kHRcdi>QRE9_O{)a~BbUU?lx|*fqn-6Z9PH$~~IViJ?f28@kgh)??K;Vo}6`uzQ*1M+K0Zz zTsi00^)=?pi9h-p`yYru`Wky6Sa;Z1gA{GElAU`0LQ>D46Ysf3E)&8!X@uE`yMKXo z1M38aFdDs&!D1bu_dK}!7i_FAN{7-J#rJzGT|Z>?+C8^^2!~WDQg`11>IzW$ajGlI zuw&vsw;d4w5eEGDpI8+Cxh@mZ{|%uc z{v!nb_YVX*hD(W#F&F=>kz&`*;ZhEJ-%>j#O6C5^;{5Qi#(CNA)|Egy_PotcOz8OD z#6*dE%(7T-vj*WBU>mWr3s|`ZVGbDIwPm=FyLm+8Khxq!)A;YL6(R-_|9gSiajgI8 zCH{|06msi`|JeU#jTf>vji}tZiDK?1tpD-SO#Ekhg82X~z21nl*7X$`mjiy!_6_B; zxSt)HD8x|*2>$18>R0jhv0{8{pPKC-(`kbL3B(cW<9MIzPvaelJ0H0tQ1F{|lU_4&$ zPcktevjhexV1UMsp1W+W-wb#dUH+y$40Njn${m&Rdqoo?T3)_PA0bWGu|5P;nhAN3w0I;Z2F6w4_@u^*+}{v4Wu*mKi{_2Xe@CfF8Ml195_ zeDyt5`LcJ0OWgm$bbiOe z-kfG|r?Nb+nEJx5SlZ|Vp1+dQs*ToEIXFxBFc`nX- z@jDWi(d(bFXBT-7VE-H6>jceCJR1Q1*a3I#bH#G^lcVEJyZ1hogUkZiXFdTNnKpe#-vg249*RTOZ<41?1?H~EJ!j=Gpz!b;ud?AF+NP{Xwkewzv*iVf``tp>1J2i~BI{89JdJV8QO!F(w$`n&gP(Ji&fs z5*x-@z7)!*kYDyA&3%oOw@My<%i9Na`hRhM{a?TDH&F1``NTklIwcu6qxZc32iVVm z*QfLK`8soGPob}eHTeCgLS`d^QDplC{9e8_?Ip z`rp|>^zGVM&ktjr?@%GNbx?J;f!5YhC28h5U!CsJ(tFZTmnLr*z_+2_sHammczfJY z$EVsy^&WSwH;$q1*8ALV7{osJ(Lze6M~l=;_5+bemBPL`<~@z}i7qR6oW>sd)Y?4$ zUU*X41*wgLO5$k0sf~CS-@u*_cYizD9x?RqSl6dfSJ|7B*jqV&A+>&+t@R)Ao!FYkE3~p+0fBWuBoo%)K1!u#iLS@rX!*nt!A060UXR6Wt z+q*A0owamo@T1eij}0DvT<2J2>%?*F`G}@9h79WV26q2CdXJ9R-bfqmjilG-h}E<= zb}#Ii4usSbY-@zNZqqoD#yVZ%o&_9PysdHg5^SuzbbkZ5L+ok&wRq6dfoo6)=(pzD z_5X9t^nX_1|KW9W{ZGsPL(BhztbuO%f0%2n|7orNX|4Zht^aBHe}M0wYjXY1kCXq0 zzRUbSMEgecI-kl22G2%Dbh^8)$Mpdb9ob%}jEv?h+b8nT?K^aOch<%j>OGfwd*MG~ z$9C;MqQBoR4`RGw(DNTLIB3_5r*b~mzrTQUy8itIoY&QHhW`l8@9H?ie+0)@bR2x( zY2Oo=2j-ZVj>9~#j>8(L9 zf5@5sAL1QDCHjDH{htmOGdqV$^Z}8;8lW=#LCg;g>9Jt(1CdD}oxUIp{}0-q1I9eX z4+MQVWH8<;{va}&hYH;{4Z|12rXtfg7R&W8oqDYh*1BMfv=i~14~=15xCiTDW{1ZM z-P-@d2*!mc3tcx3t8Vyzpq~dxGslIQu6;dtw77mo{6BPR{|^)SF8Y7iG`isbfj%BO zZ=jEdO~u%C^#8C)r_m=W$LTr{N1qQdyiea-F_f_@F|Pfec3^Cnej>!*1HavaH9>sO z?+0QH#=_nBDLp>SwL`JBVeQ|j4{5=UFnp)YU+B2W@CSkUn}zXVd*%L5#n(W?{xp65 zAczCBb<_WXxrR{tRRETh#}@OeFsa9AvCL2VRp{~jD)iOytI)fIUxl3KS0U&5Rp{v| zYQGA7Kc;>aTH_zEfGg~O`Ym((qqYC3HU7~W|ImBD>HD=c{?Qu$XpMig#y?u)AJ^*m z$6L4miTj>p4-|bN)Z6|I$+XpZsgV^f-a%^HN`9-Kg{_;B&aT9|`-H*k^jD z>_@7z{`<21KMDBEt-l5g<4wl;Z~9f0_1~_KRrD`A-2Fd)@Adz&zB}Vv|DEOjpW5}` z7S?}{m6GWH>-|60WRYvX+5gw;zXx#siE++}i~;EWzp?&X)_u!*Z+#95)`8`29>bdN ze39p%bfd4&wO?_p57%d(VC^^h@&!Jd3o;Jtzp<7pj{QHqJpTmiw0p3w`h0W~_W$%3 zxbC|rI8p4m3HyJ%G;{qo)4A@CwD>)qr4q-wNUp)^!MdyGu?$UDl!b=)EJ-}Tz>^;r9jy+5*+ zoa@8%+HWtOSj}@`?Dd-pp|xZB-5ka~n5O{761es|zA?M7=O>G2t82bjzUQYiHB_h! z4L7{!r^~nIyYqUCZDQ<_V+W1wMd|j99drUSvd!#i$zHxDOz%7K)}ZUPrQG8JEa9FO zy~d9B*weVMJ{@Bc#FN@RDx36J1kPmE>(kNQ(0f#J#(FW?qmr&!FUE5@s(VzFw?`$= z^IpAXEa$CH=i1Z7*Npx6?NM3QKxUHe*7!%u|6}X?kJkAgXlu02|G=gi z)Yq-^KU(L1wERD|&i}Adwc{V$8)y1}VZV97_apqk_j7sWody3ryGhUvVbfyd(QxdA8!o zi$UzA#U9w=!X8*0r^7w4-1mw zeWN@#i0i(%k5>Edz&={-w*&iVwa<=(V{r`1PI>!mWuF|zXZ83;w_}BIK3)=gaQ|C-nPU%A$a{*KDgVPpU6 zL}_8olXvDb*F4ehk&IiG`$mg0UMXvz#0L_`E@l0b_(9^c3geMn2bH~HxS-EuZbSZY z)-(5_X7&EpajbJ1D{vobwhen@Z@@XvUYfbiNvA_^(|DggGtwAGR9loj=b3(v(xf>A zni%7hy{(xWaCS8I!fKjcn(Sp|8pkyGuH-x2p4UDuy1?TMeO#=C&m-)A)%$2Mj>~p= z=z93L@ZxF1q3c~bY1spdu~d$8W-u1Uy|Au-qiwdmIjZHUZ4cjESmUI995uEF_A(zw z?dd`#-NYVP-*{(RJNLl4y1Qz} zO+#Bo3zgBahVzs2jlYI&7%No9#v7*7)9uR-+Eh856V}+8H256w`I4%^Mqr@EMqr@E zM(h#R*tprl#x33+;u}0{+=@NKgC~wB+Z=BXG5l*`&pghB)O(0=E|VS;&KP6D^jAdR zC3W@?2Z*<+Abb&E4{^H7(&NDytVt7J1dERczx6%|nr+K9^Iog{2h?1||H+wF`|oXS z|FI7!Z9w`yVSA3gQ0VuBd;g;r`+ZfPCw?C%?Dw&4rrTQAkWZ6FaG?42- zh4!D}|75si`ahw+Gy0{h=l?`}pV1Er{ho>clXBlkQGB3iKW1p#i4QdTLXmcv_(2nY zDCz6bHrz6V_Mz(wCAoRHI6F8}=+f;!_&G!SFSTV*b+y3{+m>PYKXKE{_McAo-e2&4 zvUvdO>OKD_q?yNiUi**jy!4j5>Vn@gP18%G&l9%en8x-WeWysi;RglfL)*>g|HSom za=Gsl@pHm=y3zjQT6oTZrq-kj`kZ#ewZXR;+ki3C2TG^e{#ykfD6m!Wan?vX$?>(@ zYU~;9h^xoP$Wj*B@;yR*Z~e7+aBb?qQrPh3ru|wmu`giyy}o?STW~h0 zK3LFW=%}Y~Zfwbh@8ggaJQK9>_rWi~@7bxdA4r^@qi<`_9A^EG5MEz(sK(HT!+kdC z-aA$dzR6>KrLt9?wPXFI*v?!jz7zMMr)TL4Z9bRN``luQTxq_qPhZCqeI;oZux{hI z^sLKc8!)D^A&0u{)WyxH+c+*1fp1nm6GO+5^>Q3-g3a(97T?%cn9b!2*5LC`CK3as zvVD3swlSxy!Tb#Uh)Ejv!#VKb%zbc4?p?z(Ji9)II+%3hILLj(qmOSFXK@C3DEIu6 zq1@n1^16WnzQa6W{o0_8x6cP=L+8V@@eS}{if3XQCb0Jm`QqL)(rJT(@Sa@J&h^dM zpwoqOz&C$rs2E$DQ`I=^0mOKAZ0)2n+aT;$4~McP_NyZyRSH8k9BZ;5C;EI_Lx*z# zb%x*R#dqwmeMVtv9sTyj0()#_TeCs-%sGKKaF#uM1qRwQ zO^z=#M5|{NY0=O0YUOL~D$=6g>eb5Ex>ck#u!^+SuOcn_)m|-rZCFKG`BkK~aTRH8 zT18sdtsM(KUunv=FW^|Z3gGJy-PoFuc z!_G7J>M(QW9vya_*{8$o8P=`VeD|5%I?SE9Lx(+QcImM9%npRipJ00XtI^q~fZrV- z*lSrKh5FVi#d2c>1so0lha1JuoDYjhy9cy;Q4cmsEA7K#(&_`Po@QwIu$Z*gf)>Qn zc)omCOj`Y*)!hs&9~P6=I?&2CL(7N7q%{CqUCq$)VKHf~2dzvqw0u}hT7#g~*$gco z7L(Qn&`LK$%ZJ6Jl?SaH>kw!}f1DbHr+;dTd*rn9BuorKtMU z**|!rGS{x2JsbUkJ@-EZPR>PNu;=b(_$7PpLprqP_61JN4Rfy!^4pP!FJLCyuY#$~ zVrM#5Tc=?8L4n`t!FMWocW$$hI;xX%Wk$ON+h8=kzGmrdu1{}mv-C#l)9Y`R-j@3G z)-_9Sv_8FoX6fBfpWgar>5bK=H`pw_t@Y_`XqMi1eR}z3>D^dMkG8Q9w5Zoj*~i4? z^fs-8-c6U&yKW`)CNHNq1bR1PtUDcY^xV0L2OKlGQeXMzgiG@uy}q)>UC=BC%4bV> zAAW7DClTfl9!Izt;lDySg76avw;=pD!cm0B5Z-{0bMRvbxjtztLe9aDBji}YjR-kb zFoBTc1UDh%82cnbonM5us?1GBDs!I+R_5LnsLY*bUeu4C`Il%zx3Jb7(-7=#VeRt9 zggUQrtfgHOXPd^xs}yS|SAdO^b+D0mOI*AjA#suTNL(a75*LY{vGagq2)TqHgc z7m1I=MdBlIk@!elBt8-siI2oZ;v;d9_$ave9_tyjL9Mwhtv2Vv*lr648ynBQ+HjD# z#r9^?xaDheUM)Cy(+Y6wraHLAcBi!Ih+)JoVi>WD7)I>`E{yNF@LE@GHqR{(7~X~%W9+Ho!Xf^Ia`M|+doal{Aqmz%~L zU!VDE!LNxG;LSuGykWnY?Ihw3v4)sKtRdzQYlu0-8e$HyhL}UFA?6Tkh&jX>Vh*u} zm_w`~<`8R$Im8-b4zY%qBUtlmL9~;;XwOABM)yT)F5GG_weSY|u^Mf>=R}AXX3~h!w;LVg)gRSV4>+RuChI z6~qW)1u=qHL5v_)5F>~cf)N(lBmb|_Ci&0U57!!7x;Dp_XoDQXZ`!7PZIY`6BgR)? z)8ln)n&V5X^J&|(XWBLGnRZQkrd`vXY1g!8+BNN&c1?SxUDKXv*R*HaHSL*pO?#$Y z)1GP9v}fA2*z?x{sPn&I&He9~ziZj?)nUi96VAmoZFj!;xU0pEx30kMw$`ybj*m!P zN*ko@(FSRIv_aY)ZIHG{8>H>g25Eb=LE0W|khVt~r0vlLX?wIm+8%9?wnrPJ?a>Cs z_RPNhziHK@Z(3WM`pvoUrtQTyAAYsi-`EQ5Wvq_9uz$~bigrU=q0P`%Xfw1G+6--l zHbYyX&Cph8Gqe@j3~hxrLtCNE&{k+Ov=!P6ZG|>NTcOQ}tvt)V>CfA9zo*bQecqmX zMTgeh@3!i#1-qeMa$d7(J@>66xmxVzh85`f4R!RK{Zy$JsPoiy>O6IwI!|4v&QsT^ z^VD_fJawHqPhF?ZQ`f2U)OG4Sb)7m-U8l}d*QxW=b?Ut6dJz4quVeoHC8MA9%U5yz zKt1AGsiyVPw_fUM(eu$2=;de~y<}gD_)GnyPEt3ilhjS>Bz2QIN!_GQQa7oS)J^In zb(1s1Cc* zkk95i@?jfI?2)pfj3_J0h_a%LC@ac{vZ9PAE6RwnqKqgj%80U}j3_J0h_a%LC@ac{ zvZ9PAE0Gb$aNlpVE#CJw*kleowPN&JRH@llsPlM*eTq4Qrxx}p3a(S<$y@T1d?i20 zSMrm5B|phm@{@ceKgn0}lYAvV$yf4|d?i20SMrm5B|phm@{@cOe%_~`^Pi8*ZSd*3 z$e48)^Taj~^Ta$dPs}6p#5^)j%p>!}JTgzrBlE;OGEdAS^Ta$dPs}6p#5^)j%p>!} zJervMFCbs?$lRM$iN%a{_BrfP>ct+V%Gt$yKGfSU8n&wIYDEs*qeMAye-X>fy+JH9 z_w}&M+>^sHb3YBs%)KxyGxxEu%-n;*GIRe3%gntWEHn3Qu*}?Z!7_8d1k2354lJ{U z5Ze^=80O*k!Iw<-#PMjtjwVp{F#JgMZm}%-o65lN6nPOt+?pFKOYb#99K%?s2Ya7) zzaCak1xd3#Vn3nZZM#dKi?{~iep47TYD}RPyj8 zxKu1FH48rp{$I;)0AZJH=g+i?1e!1H2qX56+V~5cXlc3 zW$^?3eRbw+S^dIUenTCmkJ=wzutG`um3j5@c{L0CbS_$ffC>kk2`e;!J+y-*M-87F z105%{>D)9z`eAd*7+;=u{d$IBXW8UhvA|u&&>v7Rw)yUmDEmEastEW z@S1Y0;d28B13Rr!r~2&Kd8M>}3Y0nVM41c)+#=&<5 z{43b-r#q`~*02q~yus&Yk)t^raI7plJYz*{C292Yq;NlP$#=r!|3&N#rVqQ4eUsAu zL1XZp%X5)}tXtZ{g&?jioLfwPQbE#f$M?d~LXh8sAK*YRUBYuJXh&y|roua;rP**y zrw6c~Zax~-<-&Q$DwK$o+~=Yycs>Xp-6&toEngVr3x{l+nT&7anrYVcN(E8A49b$h z_oBhz83*xERCg?!#KS$?Y=-nA8AYG6?HRuBTyz2fyB+Via_q6KV0N}*$KeBxa{Z91 z*lFWBYh34y>t5r!Z{a%IW#K$E?D_U+S@16xhHtrwJ-84@9~P0E>;Jbc>xS(aH;kp- zup{Y)@wgi%Vs4m%|KQrZn)x&HRLw)h9$)x&#hx^-r;O`uIFlav>UO`=-sOfJo!<9T zZrGD>!!;dl*a!c`d~YrM9Cy8F&q=v=V{KN&8P8XePQDVmGoQsCcx_Humme_hxo>{GanJqpgT_7g z(Qh#Bxt~5eZ|VK}c_W^<22a)(ESBYz^_h)E+76^`^6p_-`kC;aSkc*&EcWi1D6ZKv zS?t?0Rb0DgTd{x7_Tsudw-yKXOc&Shxve<3XGd|vp4*H0J-dn<_q?~bY0n)+d(WN4 zvgM!+_gUSBqdpM_y3nTUdbLaM=g+k{R?qvej~}wgLATg;%rfq$M|4~kdd9d6<9=#L z$7P^%j7u}_J$W6MhW;@QK2oioodY^930-7doN@2Q`E>XO^^$S$_iOdME2raP&{4*< zG43bRI*$IX8Q0CYe>tk-x}jr?>tftb4C}Zq=o{lY8TT(X>9|hl9^+DsyJb+vrJ#q5 z!+C91&&~ZhE&-inTnFQB>eX=_&`-v-GcJ|YaqZAm#zh#{5!Z3_&z;Riv46O|0!S*i zw_hGor=zOPwqmh@l}r|_iHU+WIa#o#rV7@!Z3S!l_JVcmtp#g(x?tUQTfy3~qhQ^B zd%@bZt6;tNy#?!zI|@PULALI^vv6kqpG_k`;I?_x~>cRj-O?SeaFw}(3(4q zKAIg2>wU+$NZ6|2TzJ3yDTii`q z$FEsajZfv9s^izKtHyWh_-P$KFi?%} z(($+H`1R|n@trz;hmIc{bY+-A9LrZp-mc>}Y;fZf3-P;jd_M2ScPzxeSI2MM=*G7r zp65L{@D)sc(oehA5yZ1i!aiac*X$$qAY>o07a{wIYY?)J*oTmP#I*?7N9;$)KH@rr z>>~~!WFK)oLiP~{5wefC0U`T{d4%jEZbZoboP!YFjOGGCr)28NMG<{0{p*SfAyX4Df`CswX2^>l)|LVbuJq&`qDs1MW&>I3zH`ar#)K2R^H z57Z0l1NDOXK)s+oP%o$t)C=kZ^@939y`Vl&FGL^yBgSsJus7`JHG~V4DKWv7X%!cm zm32Lt5*NhIDPPK!@}*oUU&@v8rCcdr%9Zk^Tq$45mGY%rDPPK!@}*oUU&@v8rCcdr z%9ZjJxw_+Dt{u<31MQfyBHp_)tYUt%vaKgWVm|GXvZVYdN6L?Kr2HsH%8zoS{3u7t zk8-5^C`ZbVa-{qyN6L?Kr2HsH%8zoS{3u6}-zm%^bYU+-k@YtAB0P8%+b3nhI@^_1 zwcc)4hV^8{dRy#`a-+N`H_D50qr50L%8PQNyeK!yi*lp9C^yQBa-+N`H_D50qr50L z%8PQNyeK!37v~fX8g=KrSAoru|7>Hqa;dhnnw3{Qxv=h}4N*Rn31vf>P&SkaWkZ=z zHk1ivLzz%ElnG@+nNT*A31vf>P&SkaWkZ=zHk1ivBQp6N)@@kZd(V%N?U1)@@47s% zwt1VCMLnLgZWCJ||H*gqpL{3($#?Rfd?)|Ock-WnC;!QJ@}GPs|H*gqpL{3($#?Rf zd?)|Ock*BOZq_|}-v&D%FWJ{{d0g#pG|T&XJZ9ZPy(X{8U-Fs!C7;P(@|pZ4pUGeH znfxW6$zSrB{3V~sU-Fs!C7;P(@|pZ4pUGeHS@`=8tQ)Wva4+iytOb0(4z0Pn-b6ho z@7S+%d0Oq;HOuRIJY^jrdQ1M2ujDWJO8%0s-^f4mjr=3u$UpLp{3GAUKk|+I6TY1!&SGuUjt2CP zW#_m8#~949g=TqGk0-=g(I@hUd?A0x7xIUEA%Dmh@`rpOf5;c|hkPM_$QSa5d?A0x z7xIUEA%Dmh@`rpOe}pe44sLHikDBF8JsuDTsTbr0%TGSA{Nw}6Pd>2x9l$%TGSA{Nw}6Pd>2xRXY%CMY z#xk*NEECJdGO=t@ruS2Z-r5dxEJ*lF8)lxEN9KunWS*Eu=81V^o|s4GiFst6m`CP` zd1RiLN9KunWS*Eu=81V^o|s4GiFq{bYBP9Dd*L{B#kT8?Wm=Xy_Qr8*@{wcJ%rnQQ znP-kkGtV4{W}Z3r%sg|vnR(_IGxN-GW#*Y<$;>mykC|tV88gouCuW{GHq1P8JeYa5 z5OSVO^AB@>Jnyv=&dt+dJIZqzJ5dIHFKOkp5A+n)qVT=2^}`Da2Svs(POL&(abDOD zF9b5OW@D>fv*B>fhT@tH6*#PrH_O)NhB;C9+^{yxP06`o^Xh4Dt;Q+%JI7gJSie$# zod#a1`)keH>-o;s8jjW)mTOW6>Vm>s7Fo8A=<`2r!Lk6Z0pT3$G(yhp@{9?ti?HYI zQ}bBdVAAfSIM z&jLY+^>8}WXMx~8!?QpTI`*vktio9!VSOe@FV47lW|n7x@Jx`7owyG_JzH}Ih4Hqu>)svOs(rLcC7C$%=Te^wg34iZwZc;ZV5h7im&g(WcI*JVtqf( z_K>tsltMV)gK7Fq51c`A%K|;Dlju-t>aqSJ(u=dX!NP3+NIu;6{FC9n{+aN3YnJDP z#MkxV?2-NgXcz(*t?K`CZ62&hO&69341!gzv@HUGp1imZG$G$JkdQtTWLw}vHX`(OIPi}) z&kea1YfXZnug?U@_2t9xMUO*M8aNYV?eA-uO?P!?R z;J{*K+|@9xyw7)hHD7l$Oly_{#v?d`(T!&u*F~i^9<7OE-I?0> zL`@vmMx{0$tBK?KsMN+!)WmU(RBGe#nz(y)T&X6GYo$^f%QbOaFO}L@sflBKp4xb# zCXVZ-QX5az#BuFZYUAmeIIf$rP6a=c+W2fuJl9O6Ha@oym*Dy-^6Bv5d}`xoiiwS% zE0U*D*T^|06Vu;%E%u<5!+J%1mdVuTv5u*!vrI0pL)c#>{V?cHeYVkaLqN}0j|jaS z>3zBhdcL|u==GA`r<$PWt51YpAL%{Q1U+A!BJ}!6@98G!`RWy+H$ZwPo1o{bTZG;q z>3y;ZdcOKa=;cXowh4N^I!5SiBE6@Ypy#V+gx(P8J=p|3UtJ^ghDq;VH9^l;-w3@C z())N5^n7)W&>JPak2OKhSMLbDG17ay33|S|N9c`{-eYGPJO^d(rT6&ipDUe%!n#SR zzqW%%jue7Nj~0SYJW&W9J5~sO;uD47@#BSHsZ9%<&ABUoy^6`8fSCVlCq_eRhgH_c0w>b5FoO{Rru%q4eX^yOf=3Vt^j;@5}-ug7hKy$J4SFSXUaPtT2(;NrQvUmOp z`MVAGoaf;6c3g9=gV(p>n)4mJp2jujJa~N@t~n3E>m9h}+y}33$2I3ac)bhPoCD$Y zdvVQq5MJMbYmWc$`c7PPY=_soan11^Uf+dl&WrGRFRnQr$LkN`nrFBCC!F2#f8y+x zdyTVO?m=Ith4Ik$7RE35c_L;AhO{ug(evD(OFoPjUJ~D%#^ZMY#y5?}OJF?bn~8D6 zX2D|OvS2ZBS+JP6ELcoj7Az(%3lu7u$Z_kSWH|NEG8}s z7894t-gz`~a6`W+-x@nwz2Vig0IJrdo+HAM7Z7f(x+!U-N zZVFZsHw7z+n}U_ZO~FdyreGy;Q?Qb_DOgF|6s#m}3RV&~1uKc0f|bNg!Ajz$yB5hk zXK=jLR&C+sRXQFa*v7Glrm@X89w9WZCcJDK+m;xQkTxL45d_zWae`~aIKeeyoZuQU zPH>GFC%8t86I>(439b?21lNdhf@{P$!8Kx>;2JSbaE%xzxJHa~Z20uT{7#Ewh3koLI<11A>UZ5(7c+kt!W%uq91Oyu|~jO!6M?4U=eXiu!y)MSVUYB zEFvxm77>>Oi-=2tMZ_h+BI1%@5phYdh`1zJL|hUqA}$FQ5tq2GK4y#wI<0nA3yXYM zBsR`IY16pjoA(i#R}&UBjT=k!OQroX4BiT65I+Pnh#!I(#1Fv?;)h@c@k20!_#v1< z{1D6_eh6j|KLj&~AA%Xg55WxLhhPTrLokE*@mKW2i@m@974ylh`Gc!>3`=aVA2My) zuHOMz(X?GJ(I+$814H1i*f#B4Y@2p2woN-1+oqk1ZPU)hwrS^L+q84BZQ8lmHtk$& zn|3a?O*#SUqcVu!Rzu|wLV*dc9F?2tAoc1W8P zJETpD9nvNTI7R?J1b>eGy*!iX-|5hr`$4PDt=m2qUGD|&nzoS*V&#CjG=hS)8bLzb4Idxw2oH{RhPMsG$r_PI>Q|Cp`sq>=e)Opc! z>iih{WAG93D{pb#AUYF99=LvYHP-A3&8tbbo7Tf6+G{4}4uHR+i_}ZeMe3#KBK1;q zk$Ne*NWBzYq+W_HQZGdpsh6UQ)JxGt>ZRx+^-^?^dMUa{y%b%fUYh-fXWwG172Ua- z^s#B(S)#2X7}gJYh~7|#L~p1=qBqnb(HrWJ=nZv9^oBYldP5x&y`c_?-cW}`Z>U3} zH`F1~8|skg4RuKLhB|bR7zTefA7?uk{%nrD!?3JrIWJL9n;6jt9*V3fcab&aF0!WF zMb?zN$eMB&SyS#JYsy_@O}UG#DR+@IoWnx*S zOf0LEiDi{Cv8+-imQ~8cvPzj)Rw)z9DrI6>rA#cVl!;}PGO?^?U33R@jB@wyJ*~_P ztnjR=y_b?!wcjYZAbU`$3%ut3QSyz~-1|ws@tXTK$v0ke&n5ZBYwnjM-+0Zvj^rD! zxlfUN<2Cmfl5f1`{y_4L*WBAjzVVv-^2j${b59-l#%u0}Bj0$$guPt2D4-f6} z@cZ1euB_>Az7~BHUi863mWSt7?YmxlC9Q`)h#y|~&}R2O3!h2V^RD*tysI#2Tf1I< zE!q+7TX?_iZri5c37k!0^W3YprOv%NddXh&+$(HC*1wkjB+9Mz*W$sosRM!o&4uUG zJK_8**H6<@=U-7b7S6xI`Bh_Jmz;kU4%id)^MoU~&Xle2HtpL9z^gLi$EkewuITBx zvIV~lL8}y=jeoYR{s#V?*p925{-QcP7svTn^Emeg-+5m-jw4#ivz0Pj`vu<3bGyoO z^OhC;=lQ4hF@GIb(qyn`sW7;d>12PG_8S9X^Nrgc8mapXYTs z7M~&wrpfz(HVe^{+<{Hy*D3t>LaORp*h+EN_xaH=b--5#6yfwz-Y&4u@N7|~*!AE!o zohpT(|9;(%Xx+!zaMb;Xr~9#bx*yZJAHD@V_H;j-$Y|Zyx(2_J4)Y9AMyN9gosJ~* z%GEZV>pm*8KFXQ?ry>?xOUV7<+>D4yYa48z^Aang z>GRl-ChtQ>@qAVHqlt{xN#KFDIi0_bSV8g_PGk%8O%_ejry9PAP}?TI&HN$c_XCJm zIBSmIabTn2wXt#fNn;*$zShLtym?W1n7RkgWd5+FZDU?SvC9@gr9gl7^) z8qhbusMOu$!+RUI=#r;|;p(lSO)9 z4?Qe5u92?Gy~ZeaHmh}?{Gz^Rp!;n>$_v-1D#9rj_cI>uM*?bFCYhcjCYP19J(JG3 z*H#wSuKveF|1D4db2(M^^q%$)*Me2O*VtU1W88VzfYy7@7Emi_oNwzwG_D=!t+oES zvplnt5pw2cDfY(cUQhqZna?g-4BxQ--pc9?Z;YTrcUd}Nk<4khm1#x#2^j<>{_U^cEpYm>H(wuaT=wzHy< z$Qhisxy_;vO|-N4%t)7G>$(@$Te?UWZ5dpv?xc>7gu^%=GlF9|;jg$mXSag}pMhWL z*)n_(SDaYhYa^(N6JNLvcSX-soaDlN6lYw@*>o1pCJf?i!itk#Nax|&m<=#teI5Tju@0j1Xj2_N`)AVqjou;SH%>=z@lk^5Wdiq>W*?814D zHpWl#ZSY-l2hXwFdx_`R@%>GBpJPu<@6+i#H)HQ5p3~<~&*=0aozAl!{pm5Cp3~_( zyV0LMsMC3t+TKe%ch8^RtJ8T-0{mP;4*vACPS5Lfo-653kLvV{PLD!v{`3KzKCRPv zPM|+Mr_;xDI?ufHrzds#fKKNbmi~0K0m0vxPUks_{`7vG&hvKmUgEii{`9O)AJOSN zU(=r+*Xeyao#$ox(;c0j)ampgABArvdFHt9oXB5=uJD}5uQ0?pkuT}cn)@=wSnWr5 zYu`#k)R{dj)13?c)kU{hhUuT9J8wBtJM)>~`Sk~*!d)263gzw<1281dvo<<1j2?Y7&hY5fanr*+zn9o4k# zLfUgW?e^QNX>p`ohjGd4I(Wt;q1sc6mL9YmV*m zdK}jr+vW8Ht~s{L>q%U5Y?s$lxaQa{ueafvW4pZGj%$wX^7>X>b8MH_)41l?F0XIH zHOF>&y#v=A+vWA`xaRmPuXo{^zF@uzKQQ0Jx|lEObulPM^heJ;kFnjBPA*R;#onlo zuHLP#PS)t1PbWEkPFYe1Mdzr0qI1+g(K+g$=p6M=bdLHbI!FBzoumGV&QbqF=cs?8 zbJRc4IqIM29Q99hj`}A$NB!&PnMybZ^Ybm-19n$+O!SDD=ju@v_atp4I##1cKHM|$ zi&!VRM7Xqmc^-6S!dL_C>oqA%1P(HH8D=nHj6^o6?fpNKu!YkiV!FYL8G@eaix z(Tga`?CM3e?Imd|(U}^(@U^{6Ea}JhMK`D?q8ro`(GBW}=mzyfbc1>#xf9~8`Klg4eE*L2K7XAgL)#mK|SF*6wF)TOl}uXK5-S}iO4vOa=G%aw#y}L zCAv`~f3(YmChI4`1-8RQ_S6HBJ@r6jPdyOXQx8P;)B}+{^+04#JrLPb4@CCV1Cc%T zKx9un5ZO}?ME2AJkv;X`|KizO*u(hvk2xL)pL>yKSFY8*n53;l<~4FfUrcD`@IBM6 zDQA%_jv&fcm7THqHB3sH?WJ@`VY$<1vE#)k-rJO~!l(Wc|au(TA&LUgN z*=!@|^SR&VzP9jm5P5TDSnX>|+DhbGBSZAHg(mS{Y?Ja7`B9c4Kgv?%M_G#eC`*wa zWhwHbEJc2lrO1!66!}q>B0tJfqb#3=k362i`cc-|*!!f<>3-fj zr~56CrSOkqCa%1y<0g`}5?R*B3*#n2llW}f9OWl6qx?i>l%L3q@)Ma+ej+o{< ziOeWJks0MDGNb%NW|W`EjPetiQGOyb%1>lQ`EjpJ8e<9{G3wI~uVQ^FJj&o(u1u=q zT#~jDdDX}S<6J_M*eW(e`G`Cy8<7WPBl4hZL>`ol$b+&Gc~CYY56VX5LD`5rC>xOn zWh3&SY(yTEjmU$t5qVHHUuXS>Js}@zuzr(r_kjk-9nG;uNn42wYWR$?MxjakGj*SQ z7haR^!fWzfcul?wugQ1eHTf>QCf|kE3Am@4{>HU3g8t3$Mv{;Wha#ye8kx z`et{7Ixpp43BPN2im_dxNsJQRC4YsVcjs-U>g-+bH{N*bDH}#DDAsxZ_RI;TpbSyj=K395Ho~d=%c1 zkHS0hQFuo_3h&5A;T`!XydxilcjTk+j(il}k&nVV@=WtB3qtWqYHRm#M&N|{(zDHF@;Vh{2j{?Ri$2b(f7 z=OO)byp$#9CFMEJM^cu2hjWiCH?KL*$a3?VbBZiCuQ`9na`T#Vg)BF(IWNd^^O|#j zEH|$?-^X(Ensa+BH?KL5$8z(Ub9O8@uQ@-*a`T#VaV$5lIq$}D^O|$4oMYEAG|v3B zu}(d`+e-4>-#+ky^m)$jNSkucH|Dw?+;3?cupP|T^BM~hFpr-pMR6TyAF`uZ+mf_0 zrIeK|MKO14g|64@*~4Dwo(ar#?JC@ni+|;(5V*&#ow}Lw{zLXYR^bgUnBo z@8G!rbSWH*l|r#(NyXAwyNfllJCVN9>Ftj7y=WeQGD6l=k;HWh1#x$f49an0+=iy?n|J%oF`HXPsLI&%lKn1?9V zY1-}CqSKQs&SrWGm25sA2E8z7q(Z6`U6WCoRwi3u`~dluehM5f&qcd45H!}@>UE_j zFNU!QkMBi0VnwGrtH7(75Z1H~O&@+;wLhhJhHy9)TZm6m#@35Ej^)GpIemWc2%iVf zx!$)OH02y&)}`UVd-OTNypHPi$tN%Fg#h(A!kKu!lE%6l>&vCx(4#$|AICTKKFk;Y z;z_qo_|dCjoabCriK4pj0UtWB%#z2mvRE^VCq3wk^^xglD~Riu?!MOwZyBv(|% zfmO^`eidmAtsv<5|IXz4UTT+%ClNtl+y|`c_T+4*l$R7UI0Kfh=2{{o|6HBYY5j!M=ZM04taEkN%yy# zpzEubgzkt(ca(I0vkAJsI!fq{d347~_iIhi_0?BGcfzAPNxI!l(Dl__LU+oeyNz_a znxN~e$As>7kM6Cc+t~zNU!5j&r#-s2k#4F9y1x2N=FJ>#GB?yQo|)2P%O1av*Sd$of&Kt=Rq&Ed*n+LNFdL z1e3`^Fr6+0Gnqm#n=J&t`qe`4H-4iK{H@Z%PI>)Wo*nj|=bxElzsGW%0>6>*zL8zwH+C)Yjhy$5-W7gh-xA*# z@V+s)!fzCo_(tCQ#-?V!aj#F-o(zUOy2H)VovBZE#G^afEZxKP>5h4H$D5^lv_9Pl zkM3l%bdS}iJLS>c)-2uQwRCAW+dZ1Mf~Msp3zcNLVf$>eFQ+^0(Y=jylNh7veX(J> zyPeDF?(pc|PP)CM`&$ju{owA)>F)CAz87>mFn`eza#|nJNJ6u`3SxvaWh`r z!$#a;FYbsDchrk}!iYQO#eKqvJMP7mj5yni3mb8c7Z)|+c6)Jm8FBl(xVsU@{XB=- zDs!K*D|25m&T;u7`k<`uSnsjUXIbC0IuClkP%qg}ai4g#UPArHZ?9I}w^Y5fT6N!2 z_0o##zD4Sc71wt@o0Imyit9Lkd!|*#)wH)(yN;u+nDraw*i>8b9ahIRT^C)UjuXr` z>!&xPE|Txmt8Yv^m*>p7QR04F>!rW<&a|FIKdbRN=VkP>zNPzF`_Rw&klD|AdG-2P zjnHnkpViDama5O{)in)$qk+1nA-X=BTB@#Ti0W6xLfW01n**}tZSG#^# zZXe~imlyVL-1UgAU*|(j^+B4f5B%71g>kn=^yMw74;E=>|Jc?6J{uCOuGimah}VnM zEmun&KujI;?1AHHO=I;H>wwk6>M1Y398Yt{s8$E7%{riItgcgMnRS3*pNZcRw_N-- zv2>}nztA=DUgGNEwuy`KTs71< z2krKsHQVj4EYWWF+xHc4x)ItTe;t6nvy6RZyz*yJe(6s)M7z%TiqNiy(GAhA(@q!K z175z(d60%^`|O8!B(&>cbwjl4)B!@f9$q&@+t+3nx+5MxI0s^4YeTes?RKGEi`!Lv zZG^VJEiANa+V8qO{0F){+z9Ot``W`o+r)1ZXB(m2;j3?iwu#|v7jv%&>o^mGxo3oN zCI)lw2;)o)<{lEpnQdt9C1IS2i`-MfI1`t-w}f%~Jo$6%n|n+cXJRk+nlR49JMKAQ zoQcQW8^Smfd%512afiKcaIBwej~Qp;EY~12&heh(c)hguk9u*O6X0H~pESngf1;`O za0_Qe7k{jA)@MJfinBfp7k-!++cXv~3ujjs3u|z8m9fyo*`~2@k@~^JS;0XQ9|Z?Z zY!n7h#G_g@|(8NZ;K@%GV2ThC;95nG#aL~lYMsd)@(Cr)_ zhOd>ORy)((&#*-MICQnPGZ(QLvyIs_M)|O{KK`vPMvZv&xNc{zGDeyB*EB{gg4-tk zne8=K+Zu}g5u#4f=p6T1YbOzaYzGO;KiX)ij>`q3-`RLi;0&^#4q(GV!Dl+8vAFw~1BK z<}oowu*Ad{!4eZ+1WQbO5iBwBMXe_w{uB@`{o4C@n?Jt6dCa#!m2@?~<{!L5}`!_K`?BB!$v40a2#Qsf85c@YVLG0he z1hIb;6U6>aOc47wF+uF##Dqre-^333L;I*znfs607mqt1`XAtvr1AAJKmOxCVxJk$ zJ~_s2*3FHKLoUzutM=Il?ZxWmnl{BMxAjfiH|?{Le0TWf2ZZ0I?MvNk+N{{7X|H0N zroD=7n)WKTY1*sUrfIKYo2I>rZJPEfwrSd{*rsW(Vwb>**%fd7t1} zOig#NN$ke7C7I8# zJzeCU1ewopyg1HrnEm5#H&vH4xwdC_?YcnG@5OAVq4g%qvYl#u(+KUw>YJR$Q?_GG z{cePI$0GV|)@4#Bm^v@IZR)$|wyE!;+oryYZkzfpx^3#a=(efvqT8mvi*B3xF1l^% zyXdy5@1onLzKd>~`o3J<{wLN6@HzP+`)Tkw`O8h!4NdMDU6y`Vb+D1WSc~ak#?wKz z)l401g!W=OXzI6#v8HZ{9-6u-dT8pV=%J~bqKBq#iXNJ}DSBw?rs$!mo1%xNZi*h7 zx+!{S>Za(SshgsQrfx1*4>@Q2jL~N}xon(G8he+U^L^p?_Al1MS?UTrv|>+Sr6J!+ zrmk4nuPn4pJruk$^+)u?)E&_iQ+Gs9Ox+PZF?C1u#MB+p6H|9YPfXnrJu!7h^u*L1 z(Gyd5L{CiJ5j`<=XSsUv$M9tu{d{C@?-g*$w7;Zh^K5GhUrqiCUrqiCUrqiCUrqiC zUrqiCUrqiCUrqiCUrqiCUrqiCUrqiCUrqiCUrqiCUrqi?8^`3o*~WPUwnsf(%+E|E z7c*=whq3=&jQv;6#xVBZVUGRh79aZ$Th+FW*oEKU4DEt(rmq;vk8!4-7s`)urjHiN zk8!4d70Qosrf(F=k8!476UvWqrcV;ek8!3y5z3EorY{i6k8!3S56X{mrVkFvk8!5| z49bsjrtb^Nk8!5o3d)ahmgmdDhA#`Qvy?tv(%t_X<#XlU6UQz1hmBpZlJpZB`%5di z7V;gN-phT*&d~I3hVcnSJsQF})(EwK z*z!HEht*SIrdhjQel6A!?XwxC(tTlIyU_7}Q~m$#oo{SZS9->;@7$TW9)|JQjKLvf zcS5a*P%y?A$40brh?B69cb9GGE}KLhj6E<0V;kEbNMSN|Ny2W^CS|uROS4;A^#heE zh3rNttESjU{ZMVREhJhsYV#pdS1QW3(RS6Xs#OyAdCobn=bd42Her#PZAOAK^PKnm zx##`8_uPAZ@42?WF}+@_ss9kxA4Gep#ItKF&GjNJVV!Qii+cuZb+aCH<+PXaPy4|e zC-!5#(UVJ0&6fPISjYEm&!nC$h5u}HyhH5NS-gWhl?;mW*k;u9`BDnYA(f_QO0nRG ze}6!2c0LFa@6JDwn-5}G=(V&V(i1~Jf;N2r#|YM$^|8*ZkNvxc-8$E1^{~cl1i$N$ z#-hnWJ@fcIo+1qKnPOd7uD8lGh{R9_xAiv66Xj76on`BmHf>3z z!!FMCJ|9*DnWF@iBJB`evFCquBGun6-tU zdMyqcmSY;M2s>DB9Cq|sbXnL|{?&&C z=_A%*E-YpJU^2zJkkXF*WI3j6KLI_4^`_ZQSl1W?cZ3~DmP0<Lmei^Y~*hk7bu{`wEXamR4yBc7>A?%a4Qha7Qc(BYD*$*1RB7NdxO>$X> zJ6IALW*9rn3plqdN$j5vlZAL9H4;zQ{$s6gFIXCSRww(G4{E|b*ACb0e*r;fz$Pg(X;Da(E;ZP*WcFvblj!+zM%QjCLt!y29#;C_&dai3j>G|`v= zSC+fPvQ-7c@a^Op%-lhHu!fYrr(eRly94GWVZ(Av!`stl|C)py{aC*u4W^nb` zMV6a>a{ce{&U8nL<&bas_353fGf5xs0l^yQ{GM7F1dS=feg@mewxUauTQaG17wah{o3>`s z8T-9yIlfzdK(Xb&$?@OP;w^B$q6t1z++$sM)5~65v#ve+JD;@c*mK=_bIyqNcsvlp z*g-3B9%ejoevjRab>y+`J2)Q6NGvxby~^MJ!}gwzB{AL&m`);LF~Ij|Lps2sjzxdz zk2~X++7!+-{lcpJrAJ`ji*al|7tEH%FBJa}&x(IN!X78()ny)lZ^Ez1(*J8H5=eO)a`s**^Im?&$ZtBP0+KMq|+mE0w zvFmnrTWD9}yJ{bMs~bL`9orPUPG{F;>W7li!vhaPi-707IDPz}d z>{fw+YP%L^x6;_P8@m=TQf-%UcFT?38e^9ME7f*MXO}W|EygYhma6U2&aTPWWsO}L z3{~5O&Msl>(#9?XW7T$nvx^zK(Abf|YCF%_`Nqz&OGXnfgtNlwWAjhWhQV5MZrF_V zVBf(xY4h{J?0+!*RxtbTCiQ23glp!-Ll=v?4!vU5#(Rw1e%EjLEpq#}~QV;2K#_{mfB9IeNk!Ko%WE@ z`keNF(sny-kJ5&mwpVEdr|nnTh|@-uHtw_urA;~Qh|+qUwhbEI)H2I?M8|E@{K?t% z(V5vX+&}rI`CF)cKl>ZFUjMO<+kVE9-3R=lWA$j{RPpgcUoReue6>h!?>)-47KvMM zeY0^({&dXRxV&|7TM?Hv;#Oj!Myx&{vHE~xZ#&p?v05|F#cItwX-CJsXg_a^9(DbA z9R0XI@{=ORQpd$ZhyKfGI!5~fyH}t0XJ5q}$@!7-#$z$|@1x9-v6r5^ zetvxJl-c+8x%t%0kC;zm&z&*0oo;N*{CMU$W4oR6(3x{4zuo1VIq}>Xli$zSI(OdW zSH#x&b0&WW=fU$f9xCR+m(QF0oty{VJl5*!H}l}jZXN?~lAm??b{?E-MW`Y6P9pYZ zOziDu?0vwT$nn(kp^csLocN25t%}&GF(=9#6S!^TJSa@ph^Ib@r#@%TdBw$3%{-~A z*4%V6aa3!5(Xn-}i!B{T+gxnvIO=t=^-<h|Oa2 zlGr5O81tC2B0g|TDK=?m!{bWchrzfEoyKEH-iJX=#|`hppr+%7_hC@$b!B<%!}~C( z=@{aD7}Ru}@IDM`I!<^W1~on3@IDM`J6#(*Cggn>)W)60V>8}|K~2XF@57*`?=IyI_uB=%}tzh{Zht-Vob+G1@nj<;qX?( zMFsOX&yu!uT*!OS>2n-Z4vbu0+?Iu?X`9SgjF^E>LC0L4`+GIi!~pRLXr*PEnSmU)5UQd}(*H6UeR`6KC+6{Qr`#jB{@TM6Q z-ZX>4n`TgW(+mo4nnB@BGbp@i28B1xpzx*{6y7w0!kcDLc+(6DZ<;~jO*6j-jsb-)@i&> zCk$z>gdxq9Fr>K>hBQ~gkmgDl(p(8cnk!*Qb0rLEu7n}Yl`y2a5{5Ka!jR@l7}8w% zJif!T@F#ELdawAP+l@D`S@GOuMqRfOn_I!n4luF9F~RG+($*T+A0FT7eT!yAxX`Q! z7n&8}LbD=VXjX&^&5CfLSrINYE5e0lMYzzc2p5_a;X<<_TxeE=3(bmfp;^(#Q^#&% zpCZ1v6>Rjw@BPm2y#6n3t#f|o@s#WfGz;Q$&4Kt_b09v~9Ei^~2jX+hf%sf=AU@X| zh|e_#;&aV`_*`=!KGz(G&ou|)bIpPHTyt=i`vN=zJ;mb|JOe#>6UQy$ix2Df?eOV# z=TrX8N9t*JKIL(Xj%Dp@@vHW=_*MH_{HlE|e$~DfziMBLU$w8ruiDq*SM6)@tM;|{ zRr^}}s(mef)xH+LYF~?AwXbVP^%*mo1xq{JWEuYK@Y>?J(3fNS}E_#1`T=zYD! zq4uNrOZ!m#rF|&=(moV_X&;Kev=7B!+K1vV?L+aG_M!Mo`%wI)eJK9YJ`{gxABw-U z55-^FhZpf|Z?k*0cd`2&N_9_}_1JlBR{KN7Rgb@ns~&$DS3UkRu6q1sT=n?Nxa#qj zan<85*CX}qtH z<4-N?G~PqV@u#Nm7v%U;YjJkGH<05`E#oxa2gvcKmUJ5L`Q!LgOFNDC_i_BGg-+wW zd>nsjfzx>39><@W=QQ4<$ML7O)oHvRlz#(W;2F;G@9DeZlp-j~Mr6VrZS?KE z@Q+9D49xQlE{#PS@fg8lZ{ojTnKyH3++({JUdOWn8J}q~kumKxUW@WMf#b_yh<#hI zB;o64a=9IR))w`Yk5Onj zYIW3YJ{OL;FV)GtpY6*Ijtq?M%O1&3Ob(BYW;gbZdwMqIH+ApbJg{l=-n+g2;n4&6i7i=nrN6(= zbs)QD!^qg^zWhXPd|;|DI&d(bn;gqc6$Ykq6XORbbAtn;xyi%h<6{$3uoyfrun%2A zem*yp-*b3hZg_O>SZ?pwL~dYYBsVshL-+O$kK`veOim0|blCj(5Z$dF!HVC;4}QJZ z`)7WI&L^~1<0%I3U#-liwD`&3pV9xs{G^5Xti}IL5&U|@*0c2q*h^&?i5C;q1RD`s+saj=SypHgs;+i&fk8 z4`RRH*0YPx#o_aBqOY;~g+iLRchTu{Rr>2r_ogfDf8_Mu zDm{kwP5o8+olc*t(tDil%~aO^1*i8`>D+Epf0cex>aWuORO>Ah9@Yfdj! z>2EuIu1b%BJNCyrSy_M9>Ah7t9;~zduhJiO`dpQM!0FzTmGz%=dT*6}&grEp{ku+| ztI~fU^>YV;Pp|pk^WKsAt8}kzbZjbb|8Ltjdbn-R;o*^?^}|D6+t9Jm$zumSqZS4x z3tpT3pZe5<*ETUWG%z*bwN2%ZP8sWggTsT6^CJ^u*y9fkVSn$n?U|hP+6KoC9?Xxz z3KdQb>_J*U751A=y76~ch)kl6wAgPwT!Kp%N2&lbAyM6hjLw= zdp72~J2q}^ALtnD+S{|x@p46Gi#irE^cI!XvN=0n)}3{wST<(ouQRR`%f{^db;edy zThzUI&tS(rdpml%Hr|tO@7%o5@p46Gi#9G~=q)O%Wpj4Etg9_5i)CYWzN|arO0jIr z&R=J2MYTnpT^k2G2k+V3KGZWb*xl7t-Eo6U+kv44z1#pB?$lYTLlt_t2%En_mhobZ z+HRy9-)Q@CD`rG=mg?9>t(S{jx0$ZUA{}Nc9lAEvN`+o-WsyF!0~LL9wOyxI)>f+s zTdAd6GIfv3bzPt5+P49^NTG^N)rD^0XH%ic%k_QvbN7DsGxxb2_e~~sD3969^IbjN z?u(>h z+JDc_^RUQ#dEPesX88rPKXwLB+jQ&Cs#>i2x!skO@sabM z-f7z-alh0r|n*L9Ja3 zTq@-^RTYq*l8(Y&#$H!(ZN{o=9Di6Uz}U(53x7wTSNDEg^5F~;Ri6j^R zvgO7Hxr9hYIdMdW^(sD%^E~~fQu0+QMQw;Pj!!gf9CuD`RAX1{+T;>E;#y3d*?32U}#f_b{WaL#K6;( zwDI(%Q7y5*fMPAgg6~pNLd<|o__S&4tLP$ECa8tWQJ9un=~^t)ID!&~mZ#-l0wLvE zo@~yiLq07psJ>9jdJYnD=1(G{{X>aJ?a63-EIN|-;>=UE zk@#TZ9kOofsoJr~X!Mzm=;)|>cx)i{joR42Kq8tXPt_)de4g^( zyW-JkwndH)r6${+iVvm6CmQAa0BW9Ljp{u$nr)NoMzl=-No}ORKORja!in+7@#r@O zqhrzd@Nq4cem^8kb8AX1B3x(;{qf*P>{w(ZeC*V~Kr}8mERY83eb><(6q>hu)SCT5 zv*)AM91xmYZdfxPR!=9A@!|exDBD?Wnx-2Xn?gx8{6ZgGNib{4?hm0PYf0w^SK`lF z((@sdWG(6Y;7S5nOOAX9C0R=z_y9_tij0g8MMStAU9|>`J059YKNd}974+YTf^XLj z#$%_(!-?qNXml(Y9vD6y?GMvC$Pent%k0L3J5H7z85tfNv%GW_P74e@Q#%qF8$1;m zj6S`8JUSNabO&F^Lg5Fa@x*X!Y)&B;Yeyre!viCcWcZ<2{3JI38ZbWSP3>*N^TFX0 zC&T{Xk(1%T34Ha2yN-kd;e&~2+>iTTxU0|GbQ=Haec`S>TGAWt?GAT_dIO%Ye^-!x zwRd^NcP~u`I}domL4T;%6Atwq^0RM9xA*jV!@EMgdpzNe13i%IIB@8&zlD)NL%5^& zpuZ~+?&w6ZJ-zy8j#GKM z#HVy4!xWQ+s0`b9zGc@dfBV@lh-U(7r;uf$^4HDq|9FKzx!GYKuMWd}MGkg{+IPt# zkC6I&NL_An;38{5_bJbj-@|~m5V28_U;ajOi^a+!Ez@RG7U_Jks3?nc4?{)X2Kt%B zM;Gdz^vq=H>ZUW9%ubJLUa*qM1u{!igK9_I~!0yQ+ zZ4%Aj!}2Py|Gz#NMQfXab{feG^R$hnx;8ErDvK_(fmrN|Zc3%LnhlT7TMWN9Z#6tI zZzE668_-U(L7g{|&(9Z-Z_byH^t_#%nTO7KZ^k-JNG4;Qwa$1mweTq+9x^qbA^$kf zV!_yxDxPhbX>nYfFFxNo)1oR10odYeX`E?nX*{Pi7A@RMrWRUZVNUgr^C|KU!|bGS$!;`Yk1r=r`SyYnsnUXxRAX!7TvJQaOk-1{D(dkqQ;$!o zXSq5wy(6in)}}dM(<`ddWLao#ZJMI`Th!(+D$UK)qFuCarqJ$%p7NTRa!1W%dAZv{ z{%NVG0~({xXTTSY#Dex)}-E1A0Ntx&M1_x_T#%&L6O zw$5#aY;nm3$U5Ar!vTGeaVXdudw0V}6XXY%$_xf#KjB)k8=f)HG8<6F zUf{Z+a&#Qp3#w9dY}!jbbF~@obY>^w(d(J>tQ0#DUF;xDV<+3E$cwnq`t)eZgP1S1 zL!SvB=v6#v)xMJMN}X<`^K>JBweB$lDX2$88ZcrQ28zviPW;Xby z8`Z{iLu12S!^Q^qsge1K=9?RuRi)Xwum)wDN}E)n&pu&S?RD0L|CL#-PLHP3qv^Cy zyr4EWJg?HW(RLL>e;rZ3MCV*@#ydybN4$IvE-YtrteAVPXfKta?Y&w?Ve@kv=KZ5! z-yBCD?Ks+i`N=@<9c1ETKc-9@i?VHTU$~&&s{Vd@^Xwm}x2a!CpCExtTQO=3WO_j* zU!Jec1X%nACd>=Xh*g>=ZS>hhOVfEJz{kB&zE4cySxVd|kiP1E1Wb&5c0c0s+8krX z{m7PDkC-oAN&hCDex$F`4=hhVm)G>$px>NIle6<_^0|4srcn*kNNf4x^`Dqa?7=tGAmX)Hd1F2lC|U>>2c|?Hwafb9^LC+sm3>6zbA4n=83W2(yOLHz z#i2PRq^Nt~OVOUIVghZ*#>92oqxs25{bYxql$@VR<#s?tY06&p9FW;a*+l-a0R#Ar1io&ovH!Gy{4wG-vWEx7FOd3{AF|%*d zY#vIR&DF|m`*~$HgnJ;yw@UXwrWP)kgXb@p19O|vpSMieW@$ostD^zODQ3?@^ZHREJAw1;lTQ{JiGp+m<8)k3)Y`{c^XTW0%xkIz@D<1 zgL4zMfEp|essYq(F0?KLf{5P>!SjI&fw{mI&@F+vP#~nX2iwyVhKuvgiPzPMK=AxT zATZYwXi-~(t?65UZ2`7LhqZ(*s;%wcPPe2lo@==PIkh!S*V`?x`KCT$3ZDOjDW^A- ze)F6=+6F#~#*Om#xtMeOlKH47GuI>o#U_QDNKDrY= z64>_8tLl#S@1(bd7INy2^bYuF`)f^8cfm(@Nj};fYEW-$Z-kFtVLqzO1X+v( zGr>885fSl}!CXZ1!7XiwvFpg(>7Uc$EqCv6vHQ)hKmKU!v6xdH#WUX*=B=5~G{Jo% zbT%|&&9qPBXB5PWXT1#CVkNK72SUMfCBdQv1@;9d0yDqLq(R?M0~45c)w-#6_>S@v z$gaaUYR{M#h?_>!6rQ7BpDzAFzexw^JD&eydC=4@@>J#TBfd#>&7f2*qZd^hb0cvNrDn{EkU3hJOsa!je)v!ooYjy+^~x{9(b2HZR&XGqYDnC z4K6KB*V>e)xJy);raWVHiE5h&sdXXq!rg6wse9Dm)Jir9xqJPAshyBp$rhJk7SGpn zh&!TY$nUcFt1>(W>^#T5#IRXoXf9L}JRhpT{9$?hTveb#-4tA#4pjrI23GX?Iak1; zRtC$`raOY?O?PN=6_8sCIn(XHZr9`-kSl|neqB@sUpO&WIx$~un0QkSnQ49Ps4p}Z z2w^U352iN?{0`uEaceY5U<@=vJqJQ-xC_r_}Fex(0PUF`t>Nnt0qwh^xw_ zR;rb0SEV{n`+<=8ADWzWFB+6>Zp{C%zzq)-fL3FEo+w`EHE7Q+gp6FlZ&EC68ZA;a zYF*T5F)=W{txcmPsz&XL8pUira-|Se(ej{Wv^*#_ju>4-%OPJu%Ym+?MQ2Q+ckgD_%+~pdMg>aY+0jVU92u_L)yAt!T!$BWyfs_9>s<(m*1g~e|Whs zwRN3|SPu4?{^8|ANEeWz!-q^w;NN*z<8K4s%K6*CF9yF^$M=Bm;Cv7GW#Dhu@x9=e zbG{e+HQ?{yd}>=Q(zaVPJL{0PZ&HX4wBjxW+fYN7*VSou)IyJq>#2pF66k3GU)Wg( zzMb>yz%K>ARmb;%U%~l4@YjNWw~k*A{yNUD2Y)^I_i#S-srV4}rMEoyD<<2-qnIDa z!qXT_ltBVSI-gRW_V9B2hC z2fCJ)16@ZpGy2<&O=%3X*aVi1O+J=y#^#~aZ!$K&`Tpbr+jFwn?xgo{>=`mzo3XGp zj|I=jZ?ZjtZ7;qKy7lkO3yYvLPS^Bn_DWT$ELw1>E<9zqUV0UAKC=OMX{Bn^;V%OJ z>MfPim6+>aKv^~x$ZvO^{>*>U#_P>_@Yg>NPWM>cs>O=^PV?7`y4LTU(=$EtFs$BH!M42m+VSmyA~XK2<4iswPP?`0X|(SZXqxen#-_gVC?vVwxCn zdU?99;4ElKZNR=l)x=j+7v|K$O4pRxrPWW@(<+|vzJ&GgnYt$?T-fI`V$OB7Ri1aX zx#kLCpUd;ayvVy=bDgbxt@5l3l5EX=_Y-R6Jzq($v=+_N>4#3gPAAV_rSo3s-1(LC zN^4hkKXCNCODs0_Y-OV9A}wv9X{(mD(R8tvCP*7>Ie9}#PTpY8$s0;@@`^)ycBA^0 zGA&K@E9F|6>Q~liX{z5?k&`#B&B+_r<>Za)OW3m_dq%;;#GX;;%V@f`p)Zr^x&}YM zp>53AY=|9pk7blTYy38!RWjHYrtb$WHk*&7@$6;6J40&gLXW3r(&KTbJ)Y_fq+!X4 zN!3OE{iO{XAVbKK#qcRwbg2lG@;4OZ_)xQC1&+7=OT~ik&GF%$B^&S(&=SGdmA3=8 z1INSIQYq-QY=MiH9FW0Jj+V-n(S<*~SSkm;26Tuf>SEL8xpF zUk|z-)b`n`>9bW&s#Q}}=Qfg=%T;*y>Y^0yX{#u8Dp*E=x)kOYr%o61TVp)yN9Mog z`11a1&B-x8?%;Ad+nFEl$;mN)?z~zb^QS*2$NU`8tykO6 ze66#e`Ft7sna_3hGv6;`Kl8oLe%1%e*w6YvXFux;*iYAPG*)k};eP#VINOCsCrU10 zze_DxSi&=dqL@>c%x0U@Z1!PIma>uWT{0G7KN;i1ROlnqJyy4hd*!6r=2kK9*~p(? zG8gCglV+?5t+r}YiRHu@D$`?k&)DrXXY6+Ogws^UU8GAF>(V8z=?8S_ z30?XRb?N`0OaBaMmC*g$w=bD72QT!H8v1^ukgt2`7;ur)75oqckNntc!2TLs;-q|3 z=al($KTh!JI$7q^eLBIX>t~ry_wOkGH3MwA^?%O)4C{*P+2_2DeR}?K_UZY{*{A2r z_Wju^?5ntreR}?K_UZY{*{A2r_FY(oeU9tcr{^zcpPs**eR{rZ-|1D@XSG{jqr{~M|^{m1^%XRG2^Ov(v&tJ|yJzutO%PQe>wZ~{N?P^^JV+!bFGoj z=^H+RIX#UzeG+pz=JT4g-Cm84JFT17&tP6h-hD6PXwG0%y)Q_O8O*XTPscUt%!1Dzv9S=v7q-Tb)r8aiMW z(?NC7eGJ(Sp^Nq@Zf)5)m-gAOb8khz2lz1E+goPO@3s~8{3?O}ruSy{!XFvMmcmn) z>cV;iFEOwd|K24-J$5JE7p9B(UJ+eSo7bm_;lzd|1Dvhk`9)!SPYp}PJ&;|dtc{nY zYi{#~8!KDF%hL6^`IZ|iYv*O@8s1!)CgOfiV~)lRjTag>bl;1{4c*f!E;Ol>+AJC^ zDblDNdtKQ*EdIU?Kk&?Z-$ux`7eBqi{_Y(s^!2Sb(AQhb%dOV77v9=hp*XfWoz{

Iw;SG$L<;L~i75X{^+=7@?u;(iFZ7d}PY7O>P zD{OeTtgcOGar+8*=?b``Kwa&eA;(by?>dXfj-`@+L9hv!M}8%v4n%sF)4puQ6GhVYH8xIq0da|)dos9rHI2)%ang*<26$UNuZ zdg-&i`PMYsx1xJt4)hu2(YGg~EGcigp^71B{!SxQ$Yq{Ka=kOR#m)P#cnHKKO&XC*NTI;Fx zxVO}e#Ks1r@$h&gIYj^HH=KxtlSBBE;Q07xB78hD7EWk?{RfHTCnJNyV}syF!~M}? zrv}6H#EbAiEFO;FuSjEw@OV5nFgy}X)FtA_>qbT%9Nh-wMD%!)W!&E5(T3LgNYvwN zXdGy2ZH=@v*ALWteK-xFrM0y_THi9z7^!a@_(aZbIb~n~cDXj^bF@DPd@qxa5^@Uz z-pimw-i4oZ;Ce43{3e(!92fobyhf^u5g@@oH+R# zj?3-+1;=mK$8&uSJ2bX zae2IaisN_bY@x{%$K^OouYgZ;+%4((0>|ZXgmV^fy_aD`SLVwr$X{4N{+SizpI<@# zJ1fY)zJmM@I4;kVKUzW0-*Oy(1DQAH{(|FZs;X-)(HubMXC{zk6|o7)LXK|ELM*YK z<0BIO36A&QmMhEuo#T&4_^TX$T*5DN z{AUvW6OKPC;n>u|Wg&|a?&SD867J#nk9~UU3E9PQRl@J%_@fg30LNd%K8pSt;`q-c z{4B>GxmPbu$kQBmHss?MIX=>wkN+vh-I6?gY0ms0;s1@}-Jh39W$N$MGYQ{4mEKy*t0&Q!C)p9KX0DU;cMj zz%Q(Tzp?^;nd23bJ^zK{XC?fn9GBzaS1aHacB8WpS$-48pO)fk8^^16=C^kz#~*Fa z$9p-xTar(3ykEk<#PKI2{1uL0l<*&M{2dAZ8;<`(!v7D)e=gzLgAtf*`SusHg}jB> zBz!%`of3W<$Ezj0j^iE)-@);_CA@>d?ff4cukOyrf5GwRrFNOwhJl52N_y6EyuUAB z&+QyPa$i2)$Z>a1KE8|NPwdaf4|81BKfv+F59G_A;`mvq-bs!>CgHOje^$bu;P{&o z{xrwmk?^KGU~@^vHoSS0K+=iiLeKv zk^cSc=wVG#Mm1TclZr|z1*A#!;xyXG2vZ(>FiGgKf&9=nS`6pbGUzZ4jqISOR(kTY z-u$eOWpQ9!A5J@FX`JPVvvc?KpBA~!*q|>k9u2WPd@_g~>4RD(-p|o^e~=;FIFVRX zoX~2{gS6V&0loCpVNDCpyGx)-uAiO;n-G<;jHcR8&;MjH0i5H>5->HZVNW)-H=Ej* zO?46+o0mxT^0Q{Su$J@k0$OgLUNY#>O9t~KgL#tvJV}42Rx?xS&l3#fDGBHW1N%r< zp9dc=2}Q?~Lk9`m?q^@L4`?F5zC_O7ae#3<4hSTmAs`ItjraEs4G$zcqXS7l(dVFu zol)F5JUX1@>HYfyRL1LT4!1V(aj-kkMNbptxy~436WR8$evZ%q6&nlV?FG)>4r6G= zALg7M3_Xa4wXo-dNgG^(m9Jc&S67laIXsS2G`nyrCs&SCM|dnY_B+vdj6+?gMv}wy zq;EN&Gj>m+{j_#yA_g=j70nt6&51uW*cUlA5=A}Kq|R7uoR^@kB9UJ>(&UJJgX}&-Tem-zO%7$}RLPD2=a8E2ToE&~I%8LNW6%K~| zc^O?v=m~}cdTt<}8}@SMu_)eaiFs!a`j6w$vHoRH0d+;j$LZ@Zp(UYBTzkgGPbG7$ zk)?fHw~W`XfHw$yDAFH$XwTR%rWktD(7ZVq4-17z3LVsg0`86l;*qiAL*c%by}Nhq z-q+O}*x$H+_mRU~1IENaGx=#QwLyT-B zn|Y}u4oS|&)#$)zB%2ZX!my$!B#O4%2gth1JOByvXVi zRZ^N!o;J(I|=DmA6o4WUl zp~vR0@sWo^v9V~lv!Q-p-<~~t_wR4-ZrU!_)6;kV{d@dcj=|--Trgbsn6+uJs7M)CMF6|RuhKx@{ z08Q)a_37&M>FV|A>hL#KCNa9``8UBPDm zeWE+qee)1rj>qD`*yuPqWh|M%>DQe&B$~JM8X}^h*ANlyzlMlt!!<;Nx2_=~oOlfp z;oED7h%UW`h&LSQT7{Tzbz=3a6KhzVSmWx%npP*)ygD)F5F87>TBoc|lsRT~qRcs~ z6J-usohWnC>O`5Nt|yAqIGW^Zs?uGhcZp4Ew6Hh(HBb@UORHs)E}?osc&}J z$L#;T$ekx14&C4F@7fd0-#z@`Ir@7i`U+7IDnx?Q2KHV-Ft7eU+-O;S`3AP1PKcn@ zxuPt^e&(RV=>>-R^oJoOK|Hgj68(d_?=$6?uk-N0#C(LS*+E&d<7ab8a77yc7_&vaXz zVxs(&eyzZjkVdyi@k4FZUq_Llm`4vHEy_P9;i9~t!{7_w=|#`-_mf8|G?cz3`Ad9+ s{UUxo1x(nkNowEk-mlR!dJt(*uYf;|#B$}2e^Qeep!@y0OD_Na0K3i4e*gdg literal 0 HcmV?d00001 diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h new file mode 100644 index 0000000000..b615e09981 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/memory_gfx.h @@ -0,0 +1,597 @@ +/****************************************************************************** + * Copyright (C) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + *****************************************************************************/ +/*! \file + \brief Architecture-specific operators on memory added for GFX9 +*/ +// reference: +// https://github.com/llvm/llvm-project/blob/main/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll + +#ifndef INTRINSIC_MEM_ACCESS_H +#define INTRINSIC_MEM_ACCESS_H + +#if defined(__clang__) && defined(__HIP__) + +#if(defined(__NVCC__) || defined(__HIPCC__)) \ + || (defined(__clang__) && (defined(__CUDA__)) || defined(__HIP__)) +#define INLINEDEVICE __forceinline__ __device__ +#elif defined(__CUDACC_RTC__) +#define INLINEDEVICE __forceinline__ __device__ +#else +#define INLINEDEVICE inline +#endif + +#if defined(__gfx803__) || defined(__gfx900__) || defined(__gfx906__) || defined(__gfx908__) \ + || defined(__gfx90a__) || defined(__gfx942__) || defined(__gfx950__) // test device +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x00020000 +#elif defined(__gfx1030__) // special device +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x31014000 +#elif defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) \ + || defined(__gfx1150__) || defined(__gfx1151__) +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x31004000 +#elif defined(__gfx1200__) || defined(__gfx1201__) +#define USE_GFX_BUFFER_INTRINSIC +#define BUFFER_RESOURCE_3RD_DWORD 0x30020000 +#else // not support +#define BUFFER_RESOURCE_3RD_DWORD -1 +#endif + +/// Controls AMD gfx arch cache operations +struct CacheOperation +{ + enum Kind + { + /// Cache at all levels - accessed again + Always, + /// Cache at global level; glc = 1 + Global, + /// Streaming - likely to be accessed once; slc = 1 + Streaming, + /// Indicates the line will not be used again, glc = 1; slc = 1 + LastUse + }; +}; + +using float16_t = _Float16; +using float32_t = float; + +template +struct NativeVector +{ + using type = T __attribute__((ext_vector_type(N))); +}; + +using float32x2_t = NativeVector::type; +using float32x4_t = NativeVector::type; + +using int32x4_t = NativeVector::type; +using int32x2_t = NativeVector::type; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct alignas(16) BufferResource +{ + union Desc + { + int32x4_t d128; + void* d64[2]; + uint32_t d32[4]; + }; + + INLINEDEVICE + BufferResource(void const* base_addr, uint32_t num_records = (0xFFFFFFFF - 1)) + { + // Reference: + // For CDNA: see section 9.1.8 in the AMD resources + // https://developer.amd.com/wp-content/resources/CDNA1_Shader_ISA_14December2020.pdf + // For RDNA: see section 8.1.8 in the AMD resources + // https://developer.amd.com/wp-content/resources/RDNA2_Shader_ISA_November2020.pdf + // The d32[3] field represents the 0x[127] ~ [96] + + // 64-bit base address + desc_.d64[0] = const_cast(base_addr); + // 32-bit number of records in bytes which is used to guard against out-of-range access + desc_.d32[2] = num_records; + // 32-bit buffer resource descriptor + desc_.d32[3] = BUFFER_RESOURCE_3RD_DWORD; + } + + INLINEDEVICE + operator int32x4_t() + { + // return desc_.d128; // NOTE HIP: Crashes compiler; see below + + /// This hack is to enforce scalarization of the variable "base_addr", where in some + /// circumstances it becomes vectorized and then in turn causes illegal lowering to GCN ISA + /// since compiler effectively tries to stuff VGPRs in slots where it only accepts SGPRs + Desc ret; + ret.d32[0] = __builtin_amdgcn_readfirstlane(desc_.d32[0]); + ret.d32[1] = __builtin_amdgcn_readfirstlane(desc_.d32[1]); + ret.d64[1] = desc_.d64[1]; + return ret.d128; + /// + } + + Desc desc_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// +/// Load +/// + +// 1 byte +__device__ char + llvm_amdgcn_raw_buffer_load_i8(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.i8"); + +// 2 bytes +__device__ float16_t + llvm_amdgcn_raw_buffer_load_f16(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.f16"); + +// 4 bytes +__device__ float32_t + llvm_amdgcn_raw_buffer_load_f32(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.f32"); + +// 8 bytes +__device__ float32x2_t + llvm_amdgcn_raw_buffer_load_f32x2(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v2f32"); + +// 16 bytes +__device__ float32x4_t + llvm_amdgcn_raw_buffer_load_f32x4(int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.load.v4f32"); + +// 4 bytes +__device__ int32_t + llvm_amdgcn_s_buffer_load_i32(int32x4_t buffer_resource, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.i32"); + +// 8 bytes +__device__ int32x2_t + llvm_amdgcn_s_buffer_load_i32x2(int32x4_t buffer_resource, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.v2i32"); + +// 16 bytes +__device__ int32x4_t + llvm_amdgcn_s_buffer_load_i32x4(int32x4_t buffer_resource, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.s.buffer.load.v4i32"); + +/// +/// Store +/// + +// 1 byte +__device__ void + llvm_amdgcn_raw_buffer_store_i8(char data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.i8"); + +// 2 bytes +__device__ void + llvm_amdgcn_raw_buffer_store_f16(float16_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.f16"); + +// 4 bytes +__device__ void + llvm_amdgcn_raw_buffer_store_f32(float32_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.f32"); + +// 8 bytes +__device__ void llvm_amdgcn_raw_buffer_store_f32x2( + float32x2_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.v2f32"); + +// 16 bytes +__device__ void llvm_amdgcn_raw_buffer_store_f32x4( + float32x4_t data, + int32x4_t buffer_resource, + uint32_t voffset, + uint32_t soffset, + int32_t cache_op) __asm("llvm.amdgcn.raw.buffer.store.v4f32"); + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// Fragment type to store loaded data + typename AccessType, + /// The bytes of loading + int LoadBytes, + /// Cache operation + CacheOperation::Kind cache_op = CacheOperation::Always> +struct buffer_load; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + char ret = llvm_amdgcn_raw_buffer_load_i8( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + char ret = llvm_amdgcn_raw_buffer_load_i8( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float16_t ret = llvm_amdgcn_raw_buffer_load_f16( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float16_t ret = llvm_amdgcn_raw_buffer_load_f16( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32_t ret = llvm_amdgcn_raw_buffer_load_f32( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32_t ret = llvm_amdgcn_raw_buffer_load_f32( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x2_t ret = llvm_amdgcn_raw_buffer_load_f32x2( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x2_t ret = llvm_amdgcn_raw_buffer_load_f32x2( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct buffer_load +{ + INLINEDEVICE + buffer_load() {} + + INLINEDEVICE + buffer_load(AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x4_t ret = llvm_amdgcn_raw_buffer_load_f32x4( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x4_t ret = llvm_amdgcn_raw_buffer_load_f32x4( + buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template < + /// Fragment type to store loaded data + typename AccessType, + /// The bytes of loading + int LoadBytes, + /// Cache operation + CacheOperation::Kind cache_op = CacheOperation::Always> +struct s_buffer_load; + +template +struct s_buffer_load +{ + INLINEDEVICE + s_buffer_load() {} + + INLINEDEVICE + s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32_t ret = llvm_amdgcn_s_buffer_load_i32( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32_t ret = llvm_amdgcn_s_buffer_load_i32( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct s_buffer_load +{ + INLINEDEVICE + s_buffer_load() {} + + INLINEDEVICE + s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x2_t ret = llvm_amdgcn_s_buffer_load_i32x2( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x2_t ret = llvm_amdgcn_s_buffer_load_i32x2( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +template +struct s_buffer_load +{ + INLINEDEVICE + s_buffer_load() {} + + INLINEDEVICE + s_buffer_load(AccessType& D, void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x4_t ret = llvm_amdgcn_s_buffer_load_i32x4( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + D = *reinterpret_cast(&ret); + } + + INLINEDEVICE + AccessType load(void const* base_ptr, uint32_t soffset) + { + BufferResource buffer_rsc(base_ptr); + int32x4_t ret = llvm_amdgcn_s_buffer_load_i32x4( + buffer_rsc, __builtin_amdgcn_readfirstlane(soffset), cache_op); + return *reinterpret_cast(&ret); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// Fragment type to store loaded data + typename AccessType, + /// The width of loading + int NumElements, + /// Cache operation + CacheOperation::Kind cache_op = CacheOperation::Always> +struct buffer_store; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + char data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_i8( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float16_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f16( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f32( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x2_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f32x2( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +template +struct buffer_store +{ + INLINEDEVICE + buffer_store(const AccessType& D, + void const* base_ptr, + uint32_t voffset, + uint32_t soffset, + uint32_t num_records = (0xFFFFFFFF - 1)) + { + BufferResource buffer_rsc(base_ptr, num_records); + float32x4_t data = *reinterpret_cast(&D); + llvm_amdgcn_raw_buffer_store_f32x4( + data, buffer_rsc, voffset, __builtin_amdgcn_readfirstlane(soffset), cache_op); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#endif // defined(__clang__) && defined(__HIP__) + +#endif // INTRINSIC_MEM_ACCESS_H diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h new file mode 100644 index 0000000000..b059db2109 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_bfloat16.h @@ -0,0 +1,251 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +/*!\file + * \brief tensile_bfloat16.h provides struct for tensile_bfloat16 typedef + */ + +#pragma once +#ifndef _TENSILE_BFLOAT16_H_ +#define _TENSILE_BFLOAT16_H_ + +// If this is a C compiler, C++ compiler below C++11, or a host-only compiler, +// we only include a minimal definition of tensile_bfloat16 +#if __cplusplus < 201103L || !defined(__HIPCC__) + +#include +typedef struct +{ + uint16_t data; +} tensile_bfloat16; + +#else // __cplusplus < 201103L || !defined(__HIPCC__) + +#include +#include +#include +#include +#include +#include + +struct tensile_bfloat16 +{ + uint16_t data; + + // Don't initialize `data` in purpose so that it could be used with + // `__shared__`, which forbid any initializer. + __host__ __device__ tensile_bfloat16() {} + + // round upper 16 bits of IEEE float to convert to bfloat16 + explicit __host__ __device__ tensile_bfloat16(float f) + : data(float_to_bfloat16(f)) + { + } + + // zero extend lower 16 bits of bfloat16 to convert to IEEE float + __host__ __device__ operator float() const + { + union + { + uint32_t int32; + float fp32; + } u = {uint32_t(data) << 16}; + return u.fp32; + } + +private: + static __host__ __device__ uint16_t float_to_bfloat16(float f) + { + union + { + float fp32; + uint32_t int32; + } u = {f}; + if(~u.int32 & 0x7f800000) + { + // When the exponent bits are not all 1s, then the value is zero, normal, + // or subnormal. We round the bfloat16 mantissa up by adding 0x7FFF, plus + // 1 if the least significant bit of the bfloat16 mantissa is 1 (odd). + // This causes the bfloat16's mantissa to be incremented by 1 if the 16 + // least significant bits of the float mantissa are greater than 0x8000, + // or if they are equal to 0x8000 and the least significant bit of the + // bfloat16 mantissa is 1 (odd). This causes it to be rounded to even when + // the lower 16 bits are exactly 0x8000. If the bfloat16 mantissa already + // has the value 0x7f, then incrementing it causes it to become 0x00 and + // the exponent is incremented by one, which is the next higher FP value + // to the unrounded bfloat16 value. When the bfloat16 value is subnormal + // with an exponent of 0x00 and a mantissa of 0x7F, it may be rounded up + // to a normal value with an exponent of 0x01 and a mantissa of 0x00. + // When the bfloat16 value has an exponent of 0xFE and a mantissa of 0x7F, + // incrementing it causes it to become an exponent of 0xFF and a mantissa + // of 0x00, which is Inf, the next higher value to the unrounded value. + u.int32 += 0x7fff + ((u.int32 >> 16) & 1); // Round to nearest, round to even + } + else if(u.int32 & 0xffff) + { + // When all of the exponent bits are 1, the value is Inf or NaN. + // Inf is indicated by a zero mantissa. NaN is indicated by any nonzero + // mantissa bit. Quiet NaN is indicated by the most significant mantissa + // bit being 1. Signaling NaN is indicated by the most significant + // mantissa bit being 0 but some other bit(s) being 1. If any of the + // lower 16 bits of the mantissa are 1, we set the least significant bit + // of the bfloat16 mantissa, in order to preserve signaling NaN in case + // the bloat16's mantissa bits are all 0. + u.int32 |= 0x10000; // Preserve signaling NaN + } + return uint16_t(u.int32 >> 16); + } +}; + +static_assert(std::is_standard_layout{}, + "tensile_bfloat16 is not a standard layout type, and thus is " + "incompatible with C."); + +static_assert(std::is_trivially_copyable{}, + "tensile_bfloat16 is not trivially copyable, and thus is " + "incompatible with C."); + +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat16& bf16) +{ + return os << float(bf16); +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a) +{ + return a; +} +inline __host__ __device__ tensile_bfloat16 operator-(tensile_bfloat16 a) +{ + a.data ^= 0x8000; + return a; +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) + float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator+(int a, tensile_bfloat16 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline __host__ __device__ tensile_bfloat16 operator+(tensile_bfloat16 a, int b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline __host__ __device__ tensile_bfloat16 operator-(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) - float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator*(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) * float(b)); +} +inline __host__ __device__ tensile_bfloat16 operator/(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return tensile_bfloat16(float(a) / float(b)); +} +inline __host__ __device__ bool operator<(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return float(a) < float(b); +} +inline __host__ __device__ bool operator==(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return float(a) == float(b); +} +inline __host__ __device__ bool operator>(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return b < a; +} +inline __host__ __device__ bool operator<=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a > b); +} +inline __host__ __device__ bool operator!=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a == b); +} +inline __host__ __device__ bool operator>=(tensile_bfloat16 a, tensile_bfloat16 b) +{ + return !(a < b); +} +inline __host__ __device__ tensile_bfloat16& operator+=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a + b; +} +inline __host__ __device__ tensile_bfloat16& operator-=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a - b; +} +inline __host__ __device__ tensile_bfloat16& operator*=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a * b; +} +inline __host__ __device__ tensile_bfloat16& operator/=(tensile_bfloat16& a, tensile_bfloat16 b) +{ + return a = a / b; +} +inline __host__ __device__ tensile_bfloat16& operator++(tensile_bfloat16& a) +{ + return a += tensile_bfloat16(1.0f); +} +inline __host__ __device__ tensile_bfloat16& operator--(tensile_bfloat16& a) +{ + return a -= tensile_bfloat16(1.0f); +} +inline __host__ __device__ tensile_bfloat16 operator++(tensile_bfloat16& a, int) +{ + tensile_bfloat16 orig = a; + ++a; + return orig; +} +inline __host__ __device__ tensile_bfloat16 operator--(tensile_bfloat16& a, int) +{ + tensile_bfloat16 orig = a; + --a; + return orig; +} +inline __host__ __device__ bool isinf(tensile_bfloat16 a) +{ + return !(~a.data & 0x7f80) && !(a.data & 0x7f); +} +inline __host__ __device__ bool isnan(tensile_bfloat16 a) +{ + return !(~a.data & 0x7f80) && +(a.data & 0x7f); +} +inline __host__ __device__ bool iszero(tensile_bfloat16 a) +{ + return !(a.data & 0x7fff); +} +inline tensile_bfloat16 sin(tensile_bfloat16 a) +{ + return tensile_bfloat16(sinf(float(a))); +} +inline tensile_bfloat16 cos(tensile_bfloat16 a) +{ + return tensile_bfloat16(cosf(float(a))); +} + +#endif // __cplusplus < 201103L || !defined(__HIPCC__) + +#endif // _TENSILE_BFLOAT16_H_ diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h new file mode 100644 index 0000000000..fe7b92ba37 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8.h @@ -0,0 +1,650 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2019-2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#ifdef TENSILE_USE_HIP +#include +#endif + +// comment out following macro to disable FP8/BF8 types +#define TENSILE_USE_FP8_BF8 + +#define HIP_HOST_DEVICE __host__ __device__ +#define HIP_HOST __host__ +#define HIP_DEVICE __device__ + +//namespace Tensile +//{ + + struct tensile_hip_fp8_e4m3: public __hip_fp8_e4m3 + { + using __hip_fp8_e4m3:: __hip_fp8_e4m3; // list base's constructor in derive's scope + + // constructor -> down cast +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE tensile_hip_fp8_e4m3(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e4m3(const _Float16 f) +#endif + : __hip_fp8_e4m3(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const float& a) + { + __x = tensile_hip_fp8_e4m3(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const double& a) + { + __x = tensile_hip_fp8_e4m3((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const tensile_hip_fp8_e4m3& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e4m3(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3& operator+=(const tensile_hip_fp8_e4m3& a) + { + __x = tensile_hip_fp8_e4m3(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + + + typedef tensile_hip_fp8_e4m3 tensile_float8; + + struct tensile_hip_fp8_e5m2: public __hip_fp8_e5m2 + { + + using __hip_fp8_e5m2:: __hip_fp8_e5m2; + +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE tensile_hip_fp8_e5m2(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e5m2(const _Float16 f) +#endif + : __hip_fp8_e5m2(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_OCP + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const float& a) + { + __x = tensile_hip_fp8_e5m2(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const double& a) + { + __x = tensile_hip_fp8_e5m2((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const tensile_hip_fp8_e5m2& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e5m2(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2& operator+=(const tensile_hip_fp8_e5m2& a) + { + __x = tensile_hip_fp8_e5m2(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + + + typedef tensile_hip_fp8_e5m2 tensile_bfloat8; + + struct tensile_hip_fp8_e4m3_fnuz: public __hip_fp8_e4m3_fnuz + { + using __hip_fp8_e4m3_fnuz:: __hip_fp8_e4m3_fnuz; + +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e4m3_fnuz(const _Float16 f) +#endif + : __hip_fp8_e4m3_fnuz(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const float& a) + { + __x = tensile_hip_fp8_e4m3_fnuz(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const double& a) + { + __x = tensile_hip_fp8_e4m3_fnuz((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const tensile_hip_fp8_e4m3_fnuz& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e4m3_fnuz(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e4m3_fnuz& operator+=(const tensile_hip_fp8_e4m3_fnuz& a) + { + __x = tensile_hip_fp8_e4m3_fnuz(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + typedef tensile_hip_fp8_e4m3_fnuz tensile_float8_fnuz; + + + struct tensile_hip_fp8_e5m2_fnuz: public __hip_fp8_e5m2_fnuz + { + using __hip_fp8_e5m2_fnuz:: __hip_fp8_e5m2_fnuz; + +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz(const _Float16 f) +#else + HIP_HOST tensile_hip_fp8_e5m2_fnuz(const _Float16 f) +#endif + : __hip_fp8_e5m2_fnuz(reinterpret_cast(f)) {} + + // operator overloadiing -> upcast +#if HIP_FP8_TYPE_FNUZ + HIP_HOST_DEVICE operator _Float16() const +#else + HIP_HOST operator _Float16() const +#endif + { + return _Float16(float(*this)); + } + // copy constructor + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const float& a) + { + __x = tensile_hip_fp8_e5m2_fnuz(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const double& a) + { + __x = tensile_hip_fp8_e5m2_fnuz((float)a).__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const tensile_hip_fp8_e5m2_fnuz& a) + { + __x = a.__x; + return *this; + } + + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator=(const _Float16& a) + { + __x = tensile_hip_fp8_e5m2_fnuz(a).__x; + return *this; + } + + // += operator + inline HIP_HOST_DEVICE tensile_hip_fp8_e5m2_fnuz& operator+=(const tensile_hip_fp8_e5m2_fnuz& a) + { + __x = tensile_hip_fp8_e5m2_fnuz(float(this->__x) + float(a.__x)).__x; + return *this; + } + }; + typedef tensile_hip_fp8_e5m2_fnuz tensile_bfloat8_fnuz; + + + // Other operator overloading + inline std::ostream& operator<<(std::ostream& os, const tensile_float8& f8) + { + os << static_cast(f8); + return os; + } + inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8& bf8) + { + os << static_cast(bf8); + return os; + } + inline std::ostream& operator<<(std::ostream& os, const tensile_float8_fnuz& f8) + { + os << static_cast(f8); + return os; + } + inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8_fnuz& bf8) + { + os << static_cast(bf8); + return os; + } + + // + inline tensile_float8 operator+(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8 operator+(tensile_float8 a, float b) + { + return static_cast(static_cast(a) + b); + } + inline tensile_float8 operator+(float a, tensile_float8 b) + { + return static_cast(a + static_cast(b)); + } + inline tensile_bfloat8 operator+(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8 operator-(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + inline tensile_bfloat8 operator-(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + // NOTE: It is not used in reference solution directly, we want to return float otherwise + inline tensile_float8 operator*(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + inline tensile_float8 operator*(float a, tensile_float8 b) + { + return static_cast(a * static_cast(b)); + } + inline tensile_float8 operator*(tensile_float8 a, float b) + { + return static_cast(static_cast(a) * b); + } + inline tensile_bfloat8 operator*(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + + inline tensile_float8 operator/(tensile_float8 a, tensile_float8 b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline tensile_bfloat8 operator/(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline bool operator<(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<(float a, tensile_float8 b) + { + return a < static_cast(b); + } + inline bool operator<(tensile_float8 a, float b) + { + return static_cast(a) < b; + } + inline bool operator<(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<=(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator<=(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator==(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator==(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator!=(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator!=(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator>(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>(float a, tensile_float8 b) + { + return a > static_cast(b); + } + inline bool operator>(tensile_float8 a, float b) + { + return static_cast(a) > b; + } + inline bool operator>(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>=(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) >= static_cast(b); + } + inline bool operator>=(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) >= static_cast(b); + } + +// FNUZ + inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, float b) + { + return static_cast(static_cast(a) + b); + } + inline tensile_float8_fnuz operator+(float a, tensile_float8_fnuz b) + { + return static_cast(a + static_cast(b)); + } + inline tensile_bfloat8_fnuz operator+(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) + static_cast(b)); + } + inline tensile_float8_fnuz operator-(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + inline tensile_bfloat8_fnuz operator-(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) - static_cast(b)); + } + // NOTE: It is not used in reference solution directly, we want to return float otherwise + inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + inline tensile_float8_fnuz operator*(float a, tensile_float8_fnuz b) + { + return static_cast(a * static_cast(b)); + } + inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, float b) + { + return static_cast(static_cast(a) * b); + } + inline tensile_bfloat8_fnuz operator*(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) * static_cast(b)); + } + + inline tensile_float8_fnuz operator/(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline tensile_bfloat8_fnuz operator/(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(static_cast(a) / static_cast(b)); + } + inline bool operator<(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<(float a, tensile_float8_fnuz b) + { + return a < static_cast(b); + } + inline bool operator<(tensile_float8_fnuz a, float b) + { + return static_cast(a) < b; + } + inline bool operator<(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) < static_cast(b); + } + inline bool operator<=(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator<=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) <= static_cast(b); + } + inline bool operator==(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator==(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) == static_cast(b); + } + inline bool operator!=(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator!=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) != static_cast(b); + } + inline bool operator>(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>(float a, tensile_float8_fnuz b) + { + return a > static_cast(b); + } + inline bool operator>(tensile_float8_fnuz a, float b) + { + return static_cast(a) > b; + } + inline bool operator>(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) > static_cast(b); + } + inline bool operator>=(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) >= static_cast(b); + } + inline bool operator>=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) >= static_cast(b); + } +//} // end of namespace Tensile + +namespace std +{ + inline bool isinf(const tensile_float8& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8& a) + { + return (static_cast(a) & 0x7f) == 0x7c; + } + + inline bool isnan(const tensile_float8& a) + { + return (static_cast(a) & 0x7f) == 0x7f; + } + inline bool isnan(const tensile_bfloat8& a) + { + return (static_cast(a) & 0x7f) > 0x7c; + } + inline bool iszero(const tensile_float8& a) + { + return (static_cast(a) & 0x7F) == 0x0; + } + inline bool iszero(const tensile_bfloat8& a) + { + return (static_cast(a) & 0x7F) == 0x0; + } + + //TODO: better to & 0x7F + inline tensile_float8 abs(const tensile_float8& a) + { + return tensile_float8(std::abs(float(a))); + } + inline tensile_bfloat8 abs(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::abs(float(a))); + } + + inline tensile_float8 sin(const tensile_float8& a) + { + return tensile_float8(std::sin(float(a))); + } + inline tensile_bfloat8 sin(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::sin(float(a))); + } + + inline tensile_float8 cos(const tensile_float8& a) + { + return tensile_float8(std::cos(float(a))); + } + inline tensile_bfloat8 cos(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8& a) + { + return std::to_string(static_cast(a)); + } + // FNUZ + inline bool isinf(const tensile_float8_fnuz& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8_fnuz& a) + { + return false; + } + + inline bool isnan(const tensile_float8_fnuz& a) + { + return static_cast(a) == 0x80; + } + inline bool isnan(const tensile_bfloat8_fnuz& a) + { + return static_cast(a) == 0x80; + } + inline bool iszero(const tensile_float8_fnuz& a) + { + return static_cast(a) == 0x0; // NOTE: only +0 exists + } + inline bool iszero(const tensile_bfloat8_fnuz& a) + { + return static_cast(a) == 0x0; // NOTE: only +0 exists + } + + inline tensile_float8_fnuz abs(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::abs(float(a))); + } + inline tensile_bfloat8_fnuz abs(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::abs(float(a))); + } + + inline tensile_float8_fnuz sin(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::sin(float(a))); + } + inline tensile_bfloat8_fnuz sin(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::sin(float(a))); + } + + inline tensile_float8_fnuz cos(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::cos(float(a))); + } + inline tensile_bfloat8_fnuz cos(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + +} // namespace std diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h new file mode 100644 index 0000000000..4c32c0086e --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/00_Final/source/tensile_float8_bfloat8_bc.h @@ -0,0 +1,1215 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (C) 2019-2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#define HIP_HOST_DEVICE __host__ __device__ +#define HIP_HOST __host__ +#define HIP_DEVICE __device__ + +// We are clipping in down--conversion by default +#define DOWNCAST_CLIPPING_ON 1 + +#ifdef DOWNCAST_CLIPPING_ON +constexpr bool downcast_clip_on = true; +#else +constexpr bool downcast_clip_on = false; +#endif + +// TODO: comment about ocp not used, just filler, will replace with hip fp8 header + +namespace tensile_hip_f8_impl +{ + __host__ inline int clz(uint32_t x) + { + return __builtin_clz(x); + } + __device__ inline int clz(uint32_t x) + { + return __clz(x); + } + + template + HIP_HOST_DEVICE uint8_t cast_to_f8(T _x, bool stoch, uint32_t rng) + { + constexpr bool is_half = std::is_same::value; + constexpr bool is_float = std::is_same::value; + static_assert(wm + we == 7, "wm+we==7"); + static_assert(is_half || is_float, "Only half and float can be cast to f8"); + + //if(sizeof(T)==2 && we==5 && !negative_zero_nan) + //return cast_to_f8_no_range_reduce<2, 5, _Float16>(_x, stoch, rng); + + const int mfmt = (sizeof(T) == 4) ? 23 : 10; + uint32_t x; + if(sizeof(T) == 4) + x = reinterpret_cast(_x); + else + x = reinterpret_cast(_x); + + uint32_t y, head, mantissa; + int exponent, bias; + uint32_t sign; + + if(sizeof(T) == 4) + { + head = x & 0xFF800000; + mantissa = x & 0x7FFFFF; + exponent = (head >> 23) & 0xFF; + sign = head >> 31; + bias = 127; + } + else + { + head = x & 0xFC00; + mantissa = x & 0x3FF; + exponent = (head >> 10) & 0x1F; + sign = head >> 15; + bias = 15; + } + + uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm); + + // Deal with inf and NaNs + if(negative_zero_nan) + { + if(sizeof(T) == 4) + { + if((x & 0x7F800000) == 0x7F800000) + return 0x80; + } + else + { + //if(__hisinf(x) || __hisnan(x)) + if((x & 0x7C00) == 0x7C00) + return 0x80; + } + } + else + { + if(sizeof(T) == 4) + { + if((x & 0x7F800000) == 0x7F800000) + return signed_inf + (mantissa != 0 ? 1 : 0); + } + else + { + if((x & 0x7C00) == 0x7C00) + return signed_inf + (mantissa != 0 ? 1 : 0); + } + } + if(x == 0) + return 0; + + // First need to check if it is normal or denorm as there is a difference of implict 1 + // Then need to adjust the exponent to align with the F8 exponent, in the meanwhile, shift + // The mantissa. Then for stochastic rounding, add rng to mantissa and truncate. And for + // RNE, no need to add rng. Then probably need to check whether there is carry and adjust + // exponent and mantissa again + + // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent bits + const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0); + const int f8_denormal_act_exponent = 1 - f8_bias; //actual exponent of f8 denormal + // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias) + // f8_exponent is the converted f8 exponent with bias encoding + // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent, + // the difference needs to be adjusted and mantissa shifted + int act_exponent, f8_exponent, exponent_diff; + + if(exponent == 0) + { // fp32/fp16 is in denormal. + /* fp32 denormal is below 2^-127 so it is usually not a concern here, we mostly concern fp16 here. + In this case, f8 is usually in denormal. But there could be exceptions. + fp16 denormal has exponent bias 15 while bf8 with NANOO has exponent bias 16. + It means that there are some numbers in fp16 denormal but they are bf8 (NANOO) normals - smallest bf8 (NANOO) normal is 2^-15. + fp16 numbers where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 are bf8 (NANOO) normal. + In this case, the fp16 mantissa should be shift left by 1 */ + act_exponent = exponent - bias + 1; + exponent_diff = f8_denormal_act_exponent + - act_exponent; // actual exponent is exponent-bias+1 as it is denormal + } + else + { // fp32/fp16 is normal with implicit 1 + act_exponent = exponent - bias; + if(act_exponent <= f8_denormal_act_exponent) + { + /* This is the case where fp32/fp16 is normal but it is in f8 denormal range. + For example fp8 nanoo mode, denormal exponent is -7, but if the fp32/fp16 + actual exponent is -7, it is actually larger due to the implict 1, + Therefore it needs to be adjust to -6 and mantissa shift right by 1. + So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */ + exponent_diff = f8_denormal_act_exponent - act_exponent; + } + else + { //both fp32/fp16 and f8 are in normal range + exponent_diff + = 0; // exponent_diff=0 does not mean there is no difference for this case, + //act_exponent could be larger. Just that it does not need shift mantissa + } + mantissa += (1 << mfmt); //Add the implicit 1 into mantissa + } + + bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) + == (1 << (mfmt - wm + exponent_diff - 1)); + /* This part is a bit tricky. The judgment of whether it is a tie needs to be done before we shift right + as shift right could rip off some residual part and make something not midpoint look like midpoint. + For example, the fp16 number 0x1002 (0 00100 0000000010), it is larger than midpoint, + but after shift right by 4 bits, it would look like midpoint. + */ + + if(exponent_diff > 0) + mantissa >>= exponent_diff; + else if(exponent_diff == -1) + mantissa <<= -exponent_diff; + bool implicit_one = mantissa & (1 << mfmt); + //if there is no implict 1, it means the f8 is denormal and need to adjust to denorm exponent + f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias + - (implicit_one ? 0 : 1); + + //Now we have the exponent and mantissa adjusted + uint32_t drop_mask = (1 << (mfmt - wm)) - 1; + //bool midpoint = (mantissa & drop_mask) == ( 1 << (mfmt-wm-1) ); + bool odd = mantissa + & (1 << (mfmt - wm)); // if the least significant bit that is not truncated is 1 + mantissa + += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask; + + //Now we deal with overflow + if(f8_exponent == 0) + { + if((1 << mfmt) & mantissa) + { + f8_exponent = 1; //denormal overflow to become normal, promote exponent + //mantissa &= (1<>= 1; + f8_exponent++; + //mantissa &= (1<>= (mfmt - wm); + + // above range: quantize to maximum possible float of the same sign + const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2); + if(f8_exponent > max_exp) + { + if(clip) + { + mantissa = (1 << wm) - 1; + f8_exponent = max_exp; + } + else + { + return signed_inf; + } + } + + if(f8_exponent == 0 && mantissa == 0) + return negative_zero_nan ? 0 : (sign << 7); + mantissa &= (1 << wm) - 1; + return (sign << 7) | (f8_exponent << wm) | mantissa; + } + + template + HIP_HOST_DEVICE T cast_from_f8(uint8_t x) + { + constexpr bool is_half = std::is_same::value; + constexpr bool is_float = std::is_same::value; + //constexpr bool is_bf16 = std::is_same::value; + static_assert(is_half || is_float, "only half and float are supported"); + + constexpr int weo = is_half ? 5 : 8; + constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7); + + T fInf, fNegInf, fNaN, fNeg0; + if(is_half) + { + const uint16_t ihInf = 0x7C00; + const uint16_t ihNegInf = 0xFC00; + const uint16_t ihNaN = 0x7C01; + const uint16_t ihNeg0 = 0x8000; + fInf = reinterpret_cast(ihInf); + fNegInf = reinterpret_cast(ihNegInf); + fNaN = reinterpret_cast(ihNaN); + fNeg0 = reinterpret_cast(ihNeg0); + } + else if(is_float) + { + const uint32_t ifInf = 0x7F800000; + const uint32_t ifNegInf = 0xFF800000; + const uint32_t ifNaN = 0x7F800001; + const uint32_t ifNeg0 = 0x80000000; + fInf = reinterpret_cast(ifInf); + fNegInf = reinterpret_cast(ifNegInf); + fNaN = reinterpret_cast(ifNaN); + fNeg0 = reinterpret_cast(ifNeg0); + } + + if(x == 0) + return 0; + + uint32_t sign = x >> 7; + uint32_t mantissa = x & ((1 << wm) - 1); + int exponent = (x & 0x7F) >> wm; + if(negative_zero_nan) + { + if(x == 0x80) + return fNaN; + } + else + { + if(x == 0x80) + return fNeg0; + if(exponent == ((1 << we) - 1)) + return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN; + } + typename std::conditional::type retval; + if(we == 5 && is_half && !negative_zero_nan) + { + retval = x << 8; + return reinterpret_cast(retval); + } + + const int exp_low_cutoff + = (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0); + + //subnormal input + if(exponent == 0) + { + //guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above + int sh = 1 + clz(mantissa) - (32 - wm); + mantissa <<= sh; + exponent += 1 - sh; + /* + exponent++; + while(mantissa<(1<>= 1 - exponent; + exponent = 0; + } + + if(sizeof(T) == 2) + retval = (sign << 15) | (exponent << 10) | mantissa; + else + retval = (sign << 31) | (exponent << 23) | mantissa; + return reinterpret_cast(retval); + } + +} // namespace hip_f8_impl + +// device specific optimized code +#if defined(__gfx940__) +namespace tensile_gfx940_f8_impl +{ + template + inline HIP_DEVICE uint8_t cast_to_f8_from_f32(float v, uint32_t rng = 0) + { + uint8_t i8data; + + union + { + float fval; + uint32_t i32val; + uint8_t i8val[4]; // not endian independent + } val; + + uint32_t ival = 0; + val.fval = v; + + if(isE2M5) + { +#ifdef DOWNCAST_CLIPPING_ON + if((val.i32val & 0x7F800000) != 0x7F800000) // all exp bits are 1 --> NaN or INF + val.fval = __builtin_amdgcn_fmed3f(val.fval, 57344.0, -57344.0); +#endif + + if(stochastic_rounding) + { + ival = __builtin_amdgcn_cvt_sr_bf8_f32(val.fval, rng, ival, 0); // 0 pos + val.i32val = ival; + i8data = val.i8val[0]; // little endian + } + else // RNE CVT + { + ival = __builtin_amdgcn_cvt_pk_bf8_f32( + val.fval, val.fval, ival, false); // false -> WORD0 + val.i32val = ival; + i8data = val.i8val[0]; + } + } + else // fp8 + { +#ifdef DOWNCAST_CLIPPING_ON + if((val.i32val & 0x7F800000) != 0x7F800000) // all exp bits are 1 --> NaN or INF + val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0); +#endif + + if(stochastic_rounding) + { + ival = __builtin_amdgcn_cvt_sr_fp8_f32(val.fval, rng, ival, 0); // 0 pos + val.i32val = ival; + i8data = val.i8val[0]; // little endian + } + else // RNE CVT + { + ival = __builtin_amdgcn_cvt_pk_fp8_f32( + val.fval, val.fval, ival, false); // false -> WORD0 + val.i32val = ival; + i8data = val.i8val[0]; + } + } + return i8data; + } + +} +#endif + +// Naming convension of datatype in hip header file +// float8: fp8 +// bfloat8: bf8 +// f8 is used to consider both float8 and bfloat8 + +//namespace TensileLite +//{ +enum class hip_f8_type +{ + bf8_fnuz = 0, // 1:5:2 + fp8_fnuz = 1, // 1:4:3 + bf8 = 2, // Placeholder, not used + fp8 = 3 // Placeholder, not used +}; + +enum class hip_f8_rounding_mode +{ + standard, + stochastic +}; + +// bias mode bit implementation +// +// "bias mode optimial" +// => "bias mode bit" = 1 +// => bias = 16 for 152, 8 for 143 +// => NAN/INF are represented as negative_zero +// +// "bias mode ieee" +// => "bias mode bit" = 0 +// => bias = 15 for 152, 7 for 143 +// => NAN/INF are represented as per IEEE conventions + +// NOTE: made optimal bias mode default assuming that's the case on device +static __device__ bool hip_f8_bias_mode_bit_device = true; +static bool hip_f8_bias_mode_bit_host = true; + +static __global__ void set_hip_f8_bias_mode_bit(bool v) +{ + hip_f8_bias_mode_bit_device = v; +} + +static void set_hip_f8_bias_mode_ieee() +{ + hipLaunchKernelGGL(set_hip_f8_bias_mode_bit, dim3(1), dim3(1), 0, 0, false); + hip_f8_bias_mode_bit_host = false; +} + +static void set_hip_f8_bias_mode_optimal() +{ + hipLaunchKernelGGL(set_hip_f8_bias_mode_bit, dim3(1), dim3(1), 0, 0, true); + hip_f8_bias_mode_bit_host = true; +} + +static inline HIP_HOST_DEVICE bool get_hip_f8_bias_mode() +{ +#if defined(__HIP_DEVICE_COMPILE__) + return hip_f8_bias_mode_bit_device; +#else + return hip_f8_bias_mode_bit_host; +#endif +} + +// data type +template +struct Float8_BFloat8 +{ + uint8_t __x; + + // default constructor + HIP_HOST_DEVICE Float8_BFloat8() = default; + +#if defined(__gfx940__) + // NOTE: ON-DEVICE... always optimal bias + explicit HIP_DEVICE Float8_BFloat8(float v, + hip_f8_rounding_mode rm = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + { + // runtime branch, use default constructor and explicit_cast() if want to avoid it + + if(rm == hip_f8_rounding_mode::stochastic) + __x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(v, rng); + else + __x + = tensile_gfx940_f8_impl::cast_to_f8_from_f32(v, rng); + } + // only host code is simulated + explicit HIP_HOST +#else // gfx940 + explicit HIP_HOST_DEVICE +#endif // gfx940 + Float8_BFloat8(float v, + hip_f8_rounding_mode rm = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + { + // NOTE: made clipping default again + if(T == hip_f8_type::bf8_fnuz) + { + if(get_hip_f8_bias_mode()) + { + __x = tensile_hip_f8_impl:: + cast_to_f8<2, 5, float, true /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + else + { + __x = tensile_hip_f8_impl:: + cast_to_f8<2, 5, float, false /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + } + else /* fp8*/ + { + if(get_hip_f8_bias_mode()) + { + __x = tensile_hip_f8_impl:: + cast_to_f8<3, 4, float, true /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + else + { + __x = tensile_hip_f8_impl:: + cast_to_f8<3, 4, float, false /*negative_zero_nan*/, downcast_clip_on /*clip*/>( + v, (rm == hip_f8_rounding_mode::stochastic), rng); + } + } + } + + // constructor from half + // no h/w inst for cvt from f16, just convert f16 to f32 and call constructor + explicit HIP_HOST_DEVICE Float8_BFloat8(_Float16 v, + hip_f8_rounding_mode rm + = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + : Float8_BFloat8((float)v, rm, rng) + { + } + + explicit HIP_HOST_DEVICE Float8_BFloat8(int v, + hip_f8_rounding_mode rm + = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + : Float8_BFloat8((float)v, rm, rng) + { + } + explicit HIP_HOST_DEVICE Float8_BFloat8(size_t v, + hip_f8_rounding_mode rm + = hip_f8_rounding_mode::standard, + uint32_t rng = 0) + : Float8_BFloat8((float)v, rm, rng) + { + } + + // constructor from hip_bfloat16 + // explicit HIP_HOST_DEVICE Float8_BFloat8(hip_bfloat16 v, hip_f8_rounding_mode r=hip_f8_rounding_mode::standard, uint32_t rng=0); + + // convert to float +#if defined(__gfx940__) + // builtin conversion + explicit inline HIP_DEVICE operator float() const + { + float fval; + uint32_t i32val = static_cast(__x); + if(T == hip_f8_type::bf8_fnuz) + // workaround: use inline asm instead of builtin function + // fval = __builtin_amdgcn_cvt_f32_bf8(i32val, 0); + asm volatile("v_cvt_f32_bf8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val)); + else + // workaround: use inline asm instead of builtin function + // fval = __builtin_amdgcn_cvt_f32_fp8(i32val, 0); + asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val)); + return fval; + } + explicit inline HIP_HOST operator float() const +#else // non gfx940 + + explicit inline HIP_HOST_DEVICE operator float() const +#endif + { + if(T == hip_f8_type::bf8_fnuz) + { + if(get_hip_f8_bias_mode()) + { + return tensile_hip_f8_impl:: + cast_from_f8<2, 5, float, true /*negative_zero_nan*/>(__x); + } + else + { + return tensile_hip_f8_impl:: + cast_from_f8<2, 5, float, false /*negative_zero_nan*/>(__x); + } + } + else /* fp8*/ + { + if(get_hip_f8_bias_mode()) + { + return tensile_hip_f8_impl:: + cast_from_f8<3, 4, float, true /*negative_zero_nan*/>(__x); + } + else + { + return tensile_hip_f8_impl:: + cast_from_f8<3, 4, float, false /*negative_zero_nan*/>(__x); + } + } + } + + // convert to half + explicit inline HIP_HOST_DEVICE operator _Float16() const + { + return _Float16(float(*this)); // convert to float, then convert to f16 + } + + // convert to hip_bfloat16 + // NOTE: no hardware instruction to convert from and to f8, may want to convert it f32 first + // explicit inline HIP_HOST_DEVICE operator hip_bfloat16() const; + + // check for zero + inline HIP_HOST_DEVICE bool is_zero() const + { + if(get_hip_f8_bias_mode()) + { + return __x == 0x00; + } + else + { + return (__x == 0x00) || (__x == 0x80); + } + } + + // check for nan + inline HIP_HOST_DEVICE bool is_nan() const + { + if(get_hip_f8_bias_mode()) + { + return __x == 0x80; + } + else + { + if(T == hip_f8_type::bf8_fnuz) + { + return (__x == 0x7d) || (__x == 0x7e) || (__x == 0x7f) || (__x == 0xfd) + || (__x == 0xfe) || (__x == 0xff); + } + else + { + return (__x == 0x79) || (__x == 0x7a) || (__x == 0x7b) || (__x == 0x7c) + || (__x == 0x7d) || (__x == 0x7e) || (__x == 0x7f) || (__x == 0xf9) + || (__x == 0xfa) || (__x == 0xfb) || (__x == 0xfc) || (__x == 0xfd) + || (__x == 0xfe) || (__x == 0xff); + } + } + } + + // check for inf + inline HIP_HOST_DEVICE bool is_inf() const + { + if(get_hip_f8_bias_mode()) + { + return __x == 0x80; + } + else + { + if(T == hip_f8_type::bf8_fnuz) + { + return (__x == 0x7c) || (__x == 0xfc); + } + else + { + return (__x == 0x78) || (__x == 0xf8); + } + } + } + // + // assignment operator overloading + // + // TODO: need to verify whether it produces extra copy, need to investigate the assembly? + // use cast_to_f8 function otherwise + inline HIP_HOST_DEVICE Float8_BFloat8& operator=(const float& a) + { + __x = Float8_BFloat8(a).__x; + return *this; + } + + inline HIP_HOST_DEVICE Float8_BFloat8& operator=(const double& a) + { + __x = Float8_BFloat8((float)a).__x; + return *this; + } + + inline __host__ __device__ Float8_BFloat8& operator=(const Float8_BFloat8& a) + { + __x = a.__x; + return *this; + } + + //inline __host__ __device__ Float8_BFloat8& operator=(const rocblas_half& a) + inline __host__ __device__ Float8_BFloat8& operator=(const _Float16& a) + { + __x = Float8_BFloat8(a).__x; + return *this; + } + + // += operator + inline __host__ __device__ Float8_BFloat8& operator+=(const Float8_BFloat8& a) + { + __x = Float8_BFloat8(float(this->__x) + float(a.__x)).__x; + return *this; + } +}; + +// TODO: place it in appropriate header +typedef Float8_BFloat8 tensile_float8_fnuz; +typedef Float8_BFloat8 tensile_bfloat8_fnuz; + +// Dummy data type for ocp (added for backward compatible header), will not be used +// Actual OCP data type will be used from new header. +typedef Float8_BFloat8 tensile_float8; +typedef Float8_BFloat8 tensile_bfloat8; + +// Other operator overloading +inline std::ostream& operator<<(std::ostream& os, const tensile_float8& f8) +{ + os << static_cast(f8); + return os; +} +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8& bf8) +{ + os << static_cast(bf8); + return os; +} +inline std::ostream& operator<<(std::ostream& os, const tensile_float8_fnuz& f8) +{ + os << static_cast(f8); + return os; +} +inline std::ostream& operator<<(std::ostream& os, const tensile_bfloat8_fnuz& bf8) +{ + os << static_cast(bf8); + return os; +} + +// +inline tensile_float8 operator+(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8 operator+(tensile_float8 a, float b) +{ + return static_cast(static_cast(a) + b); +} +inline tensile_float8 operator+(float a, tensile_float8 b) +{ + return static_cast(a + static_cast(b)); +} +inline tensile_bfloat8 operator+(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8 operator-(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +inline tensile_bfloat8 operator-(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +// NOTE: It is not used in reference solution directly, we want to return float otherwise +inline tensile_float8 operator*(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} +inline tensile_float8 operator*(float a, tensile_float8 b) +{ + return static_cast(a * static_cast(b)); +} +inline tensile_float8 operator*(tensile_float8 a, float b) +{ + return static_cast(static_cast(a) * b); +} +inline tensile_bfloat8 operator*(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} + +inline tensile_float8 operator/(tensile_float8 a, tensile_float8 b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline tensile_bfloat8 operator/(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline bool operator<(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<(float a, tensile_float8 b) +{ + return a < static_cast(b); +} +inline bool operator<(tensile_float8 a, float b) +{ + return static_cast(a) < b; +} +inline bool operator<(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<=(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator<=(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator==(tensile_float8 a, tensile_float8 b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator==(tensile_bfloat8 a, tensile_bfloat8 b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator!=(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator!=(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator>(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>(float a, tensile_float8 b) +{ + return a > static_cast(b); +} +inline bool operator>(tensile_float8 a, float b) +{ + return static_cast(a) > b; +} +inline bool operator>(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>=(tensile_float8 a, tensile_float8 b) +{ + return static_cast(a) >= static_cast(b); +} +inline bool operator>=(tensile_bfloat8 a, tensile_bfloat8 b) +{ + return static_cast(a) >= static_cast(b); +} + +// FNUZ +inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8_fnuz operator+(tensile_float8_fnuz a, float b) +{ + return static_cast(static_cast(a) + b); +} +inline tensile_float8_fnuz operator+(float a, tensile_float8_fnuz b) +{ + return static_cast(a + static_cast(b)); +} +inline tensile_bfloat8_fnuz operator+(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) + static_cast(b)); +} +inline tensile_float8_fnuz operator-(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +inline tensile_bfloat8_fnuz operator-(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) - static_cast(b)); +} +// NOTE: It is not used in reference solution directly, we want to return float otherwise +inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} +inline tensile_float8_fnuz operator*(float a, tensile_float8_fnuz b) +{ + return static_cast(a * static_cast(b)); +} +inline tensile_float8_fnuz operator*(tensile_float8_fnuz a, float b) +{ + return static_cast(static_cast(a) * b); +} +inline tensile_bfloat8_fnuz operator*(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) * static_cast(b)); +} + +inline tensile_float8_fnuz operator/(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline tensile_bfloat8_fnuz operator/(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(static_cast(a) / static_cast(b)); +} +inline bool operator<(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<(float a, tensile_float8_fnuz b) +{ + return a < static_cast(b); +} +inline bool operator<(tensile_float8_fnuz a, float b) +{ + return static_cast(a) < b; +} +inline bool operator<(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) < static_cast(b); +} +inline bool operator<=(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator<=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) <= static_cast(b); +} +inline bool operator==(tensile_float8_fnuz a, tensile_float8_fnuz b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator==(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) + { + return static_cast(a) == static_cast(b); + } +inline bool operator!=(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator!=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) != static_cast(b); +} +inline bool operator>(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>(float a, tensile_float8_fnuz b) +{ + return a > static_cast(b); +} +inline bool operator>(tensile_float8_fnuz a, float b) +{ + return static_cast(a) > b; +} +inline bool operator>(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) > static_cast(b); +} +inline bool operator>=(tensile_float8_fnuz a, tensile_float8_fnuz b) +{ + return static_cast(a) >= static_cast(b); +} +inline bool operator>=(tensile_bfloat8_fnuz a, tensile_bfloat8_fnuz b) +{ + return static_cast(a) >= static_cast(b); +} + +// ================ Explicit downcasting to support Stochastic Rounding and clipping =============== + +#if 0 // enable_if_t supported from C++14 and above, not C++11! enable it when compiler updated + template {}, int> = 0> + inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng = 0, bool clip = true) + { + // same type, no conversion + return a; + } + + // Use h/w intrinsic and optimized version when __gfx940__ + template {}), int> = 0> + inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng = 0, bool clip = true) + { +#ifdef __gfx940__ + T val; + if(std::is_same::value) + val.__x + = tensile_gfx940_f8_impl::cast_to_f8_from_f32(float(a), rng, clip); + else + val.__x + = tensile_gfx940_f8_impl::cast_to_f8_from_f32(float(a), rng, clip); + return val; +#else + return T(float(a), + stochastic_rounding ? hip_f8_rounding_mode::stochastic : hip_f8_rounding_mode::standard, + rng, + clip); +#endif + } +#else // without enable_if_t, we have to use explicit template specialization + +template +inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng = 0); + +template +inline __host__ __device__ T explicit_downcast(Ta a, uint32_t rng) +{ + // same type, no conversion + return a; +} + +// NOTE: using explicit specialization +template <> +inline __host__ __device__ tensile_float8 + explicit_downcast(float a, uint32_t rng) +{ + // Use h/w intrinsic and optimized version when __gfx940__ +#ifdef __gfx940__ + tensile_float8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_float8(float(a), hip_f8_rounding_mode::stochastic, rng); +#endif +} + +template <> +inline __host__ __device__ tensile_float8 + explicit_downcast(float a, uint32_t rng) +{ +#ifdef __gfx940__ + tensile_float8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_float8(float(a), hip_f8_rounding_mode::standard, rng); +#endif +} + +template <> +inline __host__ __device__ tensile_bfloat8 + explicit_downcast(float a, uint32_t rng) +{ +#ifdef __gfx940__ + tensile_bfloat8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_bfloat8(float(a), hip_f8_rounding_mode::stochastic, rng); +#endif +} + +template <> +inline __host__ __device__ tensile_bfloat8 + explicit_downcast(float a, uint32_t rng) +{ +#ifdef __gfx940__ + tensile_bfloat8 val; + val.__x = tensile_gfx940_f8_impl::cast_to_f8_from_f32(a, rng); + return val; +#else + return tensile_bfloat8(float(a), hip_f8_rounding_mode::standard, rng); +#endif +} + +#endif // end of explicit specialization + +//} // end of namespace TensileLite + +namespace std +{ + inline bool isinf(const tensile_float8& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8& a) + { + return (static_cast(a.__x) & 0x7f) == 0x7c; + } + + inline bool isnan(const tensile_float8& a) + { + return (static_cast(a.__x) & 0x7f) == 0x7f; + } + inline bool isnan(const tensile_bfloat8& a) + { + return (static_cast(a.__x) & 0x7f) > 0x7c; + } + inline bool iszero(const tensile_float8& a) + { + return (static_cast(a.__x) & 0x7F) == 0x0; + } + inline bool iszero(const tensile_bfloat8& a) + { + return (static_cast(a.__x) & 0x7F) == 0x0; + } + + //TODO: better to & 0x7F + inline tensile_float8 abs(const tensile_float8& a) + { + return tensile_float8(std::abs(float(a))); + } + inline tensile_bfloat8 abs(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::abs(float(a))); + } + + inline tensile_float8 sin(const tensile_float8& a) + { + return tensile_float8(std::sin(float(a))); + } + inline tensile_bfloat8 sin(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::sin(float(a))); + } + + inline tensile_float8 cos(const tensile_float8& a) + { + return tensile_float8(std::cos(float(a))); + } + inline tensile_bfloat8 cos(const tensile_bfloat8& a) + { + return tensile_bfloat8(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8& a) + { + return std::to_string(static_cast(a)); + } + // FNUZ + inline bool isinf(const tensile_float8_fnuz& a) + { + return false; + } + inline bool isinf(const tensile_bfloat8_fnuz& a) + { + return false; + } + + inline bool isnan(const tensile_float8_fnuz& a) + { + return static_cast(a.__x) == 0x80; + } + inline bool isnan(const tensile_bfloat8_fnuz& a) + { + return static_cast(a.__x) == 0x80; + } + inline bool iszero(const tensile_float8_fnuz& a) + { + return static_cast(a.__x) == 0x0; // NOTE: only +0 exists + } + inline bool iszero(const tensile_bfloat8_fnuz& a) + { + return static_cast(a.__x) == 0x0; // NOTE: only +0 exists + } + + inline tensile_float8_fnuz abs(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::abs(float(a))); + } + inline tensile_bfloat8_fnuz abs(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::abs(float(a))); + } + + inline tensile_float8_fnuz sin(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::sin(float(a))); + } + inline tensile_bfloat8_fnuz sin(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::sin(float(a))); + } + + inline tensile_float8_fnuz cos(const tensile_float8_fnuz& a) + { + return tensile_float8_fnuz(std::cos(float(a))); + } + inline tensile_bfloat8_fnuz cos(const tensile_bfloat8_fnuz& a) + { + return tensile_bfloat8_fnuz(std::cos(float(a))); + } + + inline std::string to_string(const tensile_float8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + inline std::string to_string(const tensile_bfloat8_fnuz& a) + { + return std::to_string(static_cast(a)); + } + +} // namespace std diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv new file mode 100644 index 0000000000..e4c2926cbc --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.csv @@ -0,0 +1,2 @@ +GFlops, SizeI, SizeJ, SizeK, SizeL, LDD, LDC, LDA, LDB, TotalFlops, Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 +0, 16, 16, 1, 32, 16, 16, 16, 16, 16384, -1 diff --git a/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml new file mode 100644 index 0000000000..f12bac9074 --- /dev/null +++ b/tensilelite/1_BenchmarkProblems/Cijk_Ailk_Bjlk_S_MX_B_UserArgs_00/Data/00_Final.yaml @@ -0,0 +1,304 @@ +- MinimumRequiredVersion: 4.33.0 +- ProblemSizes: + - Exact: [16, 16, 1, 32, 16, 16, 16, 16] +- BiasTypeArgs: [[]] +- ActivationArgs: +- 1LDSBuffer: 0 + ActivationAlt: false + ActivationFuncCall: false + ActivationFused: true + AssertAIGreaterThanEqual: -1 + AssertAILessThanEqual: -1 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 1 + AssignedDerivedParameters: true + AssignedProblemIndependentDerivedParameters: true + BaseName: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x1A9fya6zl43sH9vlrOEkFz4b8sLxnJYl7JpJ1ddfe8a4= + BufferLoad: true + BufferStore: true + CUCount: null + CUOccupancy: 5 + ClusterLocalRead: 0 + CodeObjectVersion: '4' + ConvertAfterDS: false + CustomKernelName: '' + DebugStreamK: 0 + DepthU: 32 + DirectToLds: false + DirectToLdsA: false + DirectToLdsB: false + DirectToVgprA: false + DirectToVgprB: false + DirectToVgprSparseMetadata: false + EdgeType: ShiftPtr + EnableF32XEmulationLds: false + EnableF32XdlMathOp: true + EnableMatrixInstruction: true + ExpandPointerSwap: true + ExpertSchedulingMode: 0 + ForceDisableShadowInit: false + GlobalReadPerMfma: 1 + GlobalReadVectorWidthA: 8 + GlobalReadVectorWidthB: 4 + GlobalSplitU: 1 + GlobalSplitUAlgorithm: MultipleBuffer + GlobalSplitUCoalesced: false + GlobalSplitUWorkGroupMappingRoundRobin: false + GlobalWriteVectorWidth: 1 + GroupLoadStore: false + GuaranteeNoPartialA: false + GuaranteeNoPartialB: false + GuaranteeNoPartialMetadata: true + ISA: [9, 5, 0] + InnerUnroll: 1 + InterleaveAlpha: 0 + InternalSupportParams: {KernArgsVersion: 2, SupportCustomStaggerU: true, SupportCustomWGM: true, + SupportUserGSU: true, UseUniversalArgs: true} + Kernel: true + KernelLanguage: Assembly + KernelNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1 + LDSTrInst: false + LSCA: 16 + LSCB: 16 + LSPA: 32 + LSPB: 16 + LVCA: 2 + LVCB: 4 + LVPA: 4 + LVPB: 4 + LdsBlockSizePerPadA: 512 + LdsBlockSizePerPadB: 256 + LdsBlockSizePerPadMetadata: 0 + LdsBytesNoAmax: 13056 + LdsInitCVgprs: false + LdsNumBytes: 13056 + LdsNumElementsAlignedA: 2304 + LdsNumElementsAlignedB: 2560 + LdsNumElementsAlignedMetadata: 0 + LdsOffsetA: 0 + LdsOffsetA_Blk: 8192 + LdsOffsetB: 2304 + LdsOffsetB_Blk: 10496 + LdsOffsetBias: 0 + LdsOffsetBiasGSU: 0 + LdsOffsetBiasNonGSU: 0 + LdsOffsetMetadata: 2304 + LdsOffsetMetadata_Blk: 10496 + LdsPadA: 16 + LdsPadB: 16 + LdsPadMetadata: 0 + LocalReadVectorWidth: 4 + LocalSplitU: 1 + LocalSplitUReuseLDS: 1 + LocalWritePerMfma: -1 + LocalWriteUseSgprA: false + LocalWriteUseSgprB: false + LoopIters: 1 + LoopUnroll: 32 + MFMA_BF16_1K: false + MIArchVgpr: false + MIBlock: [16, 16, 32, 1, 1, 1] + MIInputPerThread: 8 + MIInputPerThreadA: 8 + MIInputPerThreadB: 8 + MIInputPerThreadMetadata: 8 + MIOutputVectorWidth: 4 + MIRegPerOut: 1 + MIWaveGroup: [1, 1] + MIWaveTile: [1, 1] + MIWaveTileA: 1 + MIWaveTileB: 1 + MIWaveTileMetadata: 0 + MacroTile0: 16 + MacroTile1: 16 + MacroTileA: 16 + MacroTileB: 16 + MagicDivAlg: 2 + MathClocksUnrolledLoop: 109 + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 32 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 32, 1] + MaxOccupancy: 40 + MbskPrefetchOpt: 0 + NoLdsWriteCode: false + NoReject: false + NoTailLoop: false + NonTemporal: -1 + NonTemporalA: 0 + NonTemporalB: 0 + NonTemporalC: 0 + NonTemporalD: 0 + NonTemporalE: 0 + NonTemporalMetadata: 0 + NonTemporalWS: 0 + NumElementsPerBatchStore: 0 + NumElementsPerThread: 4 + NumGlobalWriteVectorsPerThread: 4 + NumLoadsA: 1 + NumLoadsB: 2 + NumLoadsCoalescedA: 1 + NumLoadsCoalescedB: 1 + NumLoadsPerpendicularA: 1 + NumLoadsPerpendicularB: 2 + NumThreads: 64 + NumWaveSplitK: 1 + OptNoLoadLoop: 1 + PackedC0IdxChars: [I] + PackedC0IndicesX: [0] + PackedC1IdxChars: [J] + PackedC1IndicesX: [1] + PrefetchGlobalRead: 1 + PrefetchLocalRead: 0 + PreloadKernArgs: true + ProblemType: + Activation: false + ActivationComputeDataType: 0 + ActivationNoGuard: false + ActivationType: none + AllowNoFreeDims: false + AssignedDerivedParameters: true + Batched: true + BetaOnlyUseBias: false + BiasDataTypeList: [] + BiasSrc: D + ComplexConjugateA: false + ComplexConjugateB: false + ComputeDataType: 0 + DataType: 0 + DataTypeA: 0 + DataTypeAmaxD: 0 + DataTypeB: 0 + DataTypeE: 0 + DestDataType: 0 + F32XdlMathOp: 10 + Gradient: false + GroupedGemm: false + HighPrecisionAccumulate: false + Index0: 0 + Index01A: 0 + Index01B: 1 + Index1: 1 + IndexAssignmentsA: [0, 3, 2] + IndexAssignmentsB: [1, 3, 2] + IndexAssignmentsLD: [4, 5, 6, 7] + IndexAssignmentsMetadata: [3, 0, 2] + IndexUnroll: 3 + IndexUnrollA: 1 + IndexUnrollB: 1 + IndexUnrollM: 0 + IndicesBatch: [2] + IndicesFree: [0, 1] + IndicesSummation: [3] + MirrorDimsA: [] + MirrorDimsB: [] + MirrorDimsMetadata: [] + NumIndicesBatch: 1 + NumIndicesC: 3 + NumIndicesFree: 2 + NumIndicesLD: 4 + NumIndicesSummation: 1 + OperationType: GEMM + OutputAmaxD: false + SetConstStrideA: [] + SetConstStrideB: [] + SetConstStrideBias: [] + SilentHighPrecisionAccumulate: false + Sparse: 0 + StochasticRounding: false + StridedBatched: true + SupportUserArgs: true + SwizzleTensorA: false + SwizzleTensorB: false + TLUA: true + TLUB: true + Tensor0: 0 + Tensor1: 1 + TileA: 0 + TileAwareSelection: false + TileB: 1 + TotalIndices: 4 + TransposeA: 0 + TransposeB: 1 + UseBeta: true + UseBias: 0 + UseE: false + UseInitialStridesAB: false + UseInitialStridesCD: false + UseScaleAB: '' + UseScaleAlphaVec: 0 + UseScaleCD: false + ScheduleGlobalRead: 1 + ScheduleIterAlg: 3 + ScheduleLocalWrite: 1 + SolutionIndex: 0 + SolutionNameMin: Cijk_Ailk_Bjlk_S_MX_B_UserArgs_MT16x16x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS1_FDSI0_GRPM1_GRVWA8_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB256_LBSPPM0_LPA16_LPB16_LPM0_LRVW4_LWPMn1_MIAV0_MIWT1_1_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR1_PLR0_PKA1_SIA3_SS1_SU32_SUM0_SUS256_SPO0_SRVW0_SSO0_SVW1_SK0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA1_VWB1_WSGRA0_WSGRB0_WS64_WG16_4_1_WGM8_WGMXCC1_WGMXCCGn1 + SourceSwap: 1 + StaggerU: 32 + StaggerUMapping: 0 + StaggerUStride: 256 + StorePriorityOpt: false + StoreRemapVectorWidth: 0 + StoreSwapAddr: false + StoreSyncOpt: 0 + StoreVectorWidth: 1 + StreamK: 0 + StreamKAtomic: 0 + StreamKXCCMapping: 0 + SubGroup0: 4 + SubGroup1: 16 + SubGroupA: 4 + SubGroupB: 16 + SuppressNoLoadLoop: false + ThreadTile: [1, 1] + ThreadTile0: 4 + ThreadTile1: 1 + ThreadTileA: 4 + ThreadTileB: 1 + TransposeLDS: 0 + TransposeLDSMetadata: true + ULSGRODoubleG2L: 0 + UnrollLoopSwapGlobalReadOrder: 0 + UnrollMajorLDSA: 0 + UnrollMajorLDSB: 0 + UnrollMajorLDSMetadata: true + Use64bShadowLimit: 1 + UseDotInstruction: false + UseF32XEmulation: true + UseInstOffsetForGRO: 0 + UseSgprForGRO: -1 + Valid: true + VectorStore: -1 + VectorWidthA: 1 + VectorWidthB: 1 + WaveSeparateGlobalReadA: 0 + WaveSeparateGlobalReadB: 0 + WaveSeparateGlobalReadMetadata: 0 + WaveSplitK: false + WavefrontSize: 64 + WorkGroup: [16, 4, 1] + WorkGroupMapping: 8 + WorkGroupMappingXCC: 1 + WorkGroupMappingXCCGroup: -1 + WorkGroupReduction: false + WorkspaceCheck: [4, 0, 1] + _DepthU: 32 + _DepthUA: 32 + _DepthUB: 32 + _DepthUMetadata: 32 + _GlobalAccumulation: MultipleBuffer + _UseSgprForGRO: false + _VectorStore: 1 + _WorkspaceSizePerElemBias: 0 + _WorkspaceSizePerElemC: 4 + _staggerStrideShift: 1 + enableLDSTrA: false + enableLDSTrB: false + reorderGRInstForDTVA: false + reorderGRInstForDTVB: false + tailLoopOptA: false + tailLoopOptB: false From 05d394539b0e22d2706876f15f81ec7f85abfdad Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Thu, 12 Jun 2025 14:38:33 -0500 Subject: [PATCH 11/18] tf32 16x16x32 debug --- tensilelite/Tensile/Components/F32XEmulation.py | 8 ++++---- tensilelite/Tensile/KernelWriterAssembly.py | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 566d342906..af4e7607c5 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -276,13 +276,13 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v vgprSize = width / 2 (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 683e65dbd1..847a2a77b1 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -108,6 +108,7 @@ class TailOptParams: finalLoop: int = 0 dbgCounter = 0 +dbgCounterMFMA = 0 ################################################################################ # Assembly Kernel @@ -6781,6 +6782,9 @@ def mfmaIter(self, kernel, tPA, tPB, u, innerUnroll, vregSetIdx, unrollLoopIdx = a=src0, b=src1, acc2=self.accVgprReadWriteIndex(kernel, accStart, (accEnd-accStart+1)), neg=neg_flag,\ comment="left value = %s[%u+%u:%u+%u]" % (accumRegType, accStart, accStoreCIdx, accEnd, accStoreCIdx))) prevAccIdx = accIdx + global dbgCounterMFMA + imod.add(TextBlock(str("label_mfma_") + str(dbgCounterMFMA) + ":\n")) + dbgCounterMFMA += 1 if kernel["ExpertSchedulingMode"] > 0: imod.add(SWaitAlu(va_vdst=0, comment="wait for the current iter's writes to complete")) @@ -9241,6 +9245,7 @@ def localWriteBody(tP): paramList[1] = paramList[1] - 65536 else: dstAddr=vgpr(lwa) + #carson: paramList[1] holds ds_write offset, but it is being split into two calls and 16 added to offset beyond this command somewhere ds = DSModifiers(na=1, offset=paramList[1]) writeInst = LocalWriteX(dstAddr=dstAddr, src=paramList[0], ds=ds, comment=comment) else: From 883eb8c182f455c29ca55f3f7b6eeeadc9c8b578 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Fri, 13 Jun 2025 01:13:30 -0500 Subject: [PATCH 12/18] fixing broken fp32 runs from instOffset --- clients/include/allclose.hpp | 2 +- tensilelite/Tensile/KernelWriterAssembly.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/include/allclose.hpp b/clients/include/allclose.hpp index 884321db0e..eef393d770 100644 --- a/clients/include/allclose.hpp +++ b/clients/include/allclose.hpp @@ -69,7 +69,7 @@ bool allclose(size_t* N, T* a, T* b, double atol, double rtol, success = false; double error = std::abs(a[i] - b[i]); - double relError = error / std::abs(a[i]); + double relError = (std::abs(a[i]) < 1e-06) ? 0.0 : error / std::abs(a[i]) + 1e-06; double tolerance = atol + std::abs(rtol * b[i]); if(!(error <= tolerance)) success = false; diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 847a2a77b1..18f8ef7018 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -5559,8 +5559,8 @@ def closeLoop(self, kernel, tPA, tPB, loopIdx, finalLoop, emitEndLabelOnly=False if noExit: # No exit. No dec code if decValue is 2 - # if decValue == 2: - decCode = "" + if decValue == 2: + decCode = "" condCode = "" nonFinalJumpNeeded = False if finalLoop: @@ -8270,7 +8270,7 @@ def globalReadBody(tP): glc=isGlc, slc=isSlc, nt=isNT, lds=isLds, \ hi16=isHigh16Bits , \ comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp))) - instOffset += 16 #Carson: debug 256b reads + #instOffset += 16 #Carson: debug 256b reads if unrollMirrorWithSoffset: codeMod = Module("mirrorIdx%u"%loopCnt) From 13dad1be50f1f7da756db7a672b3edaf0e3880d7 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Mon, 23 Jun 2025 14:28:58 -0500 Subject: [PATCH 13/18] tf32 MI 16x16x32 working with 256b reads --- .../Tensile/Components/F32XEmulation.py | 58 +++++++++---------- .../Tensile/Components/LraTileAssignment.py | 8 ++- tensilelite/Tensile/KernelWriterAssembly.py | 6 +- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index af4e7607c5..25dce2b35b 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -198,7 +198,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v for itr in range(int(width / numElementsPerIter)): #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + #tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) offset = "+" + str(itr * 2) aHigh1 = aHigh + offset aHigh2 = aHigh + "+1" + offset @@ -228,28 +228,28 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) - if not kernel["EnableF32XEmulationLds"]: - tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) - tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+1"), src1=vgpr(tmp2))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) - tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh2, vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+2"), src1=vgpr(tmp1))) - tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) - tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) - tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) - tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) - tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) - tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + # if not kernel["EnableF32XEmulationLds"]: + # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) + # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) + # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) + # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+1"), src1=vgpr(tmp2))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh2, vgprMask="", vi=0)) + # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+2"), src1=vgpr(tmp1))) + # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) + # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + # tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) + # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) + # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) + # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) + # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) + # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) + # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) + # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) + # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) aStart = aStart + "+4" bStart = bStart + "+4" @@ -276,16 +276,16 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v vgprSize = width / 2 (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2)) diff --git a/tensilelite/Tensile/Components/LraTileAssignment.py b/tensilelite/Tensile/Components/LraTileAssignment.py index e84a9dd70b..299d2b2585 100644 --- a/tensilelite/Tensile/Components/LraTileAssignment.py +++ b/tensilelite/Tensile/Components/LraTileAssignment.py @@ -26,7 +26,7 @@ from rocisa.container import vgpr, ContinuousRegister from rocisa.instruction import VAddU32, vectorStaticRemainder, \ vectorStaticDivideAndRemainder, vectorStaticDivide, vectorStaticMultiply, \ - vectorStaticMultiplyAdd + vectorStaticMultiplyAdd, VLShiftLeftAddU32 from ..Component import LraTileAssignment, LraTileProperties from dataclasses import dataclass @@ -287,9 +287,13 @@ def LraTileAssignmentCode(self, writer, kernel, tP, tReg, kReg, tmpVgprRes, divi comment="5.2 offset in wave: lrOffset = bnOffset + lrKOffset")) module.add(VAddU32(dst=vgpr(tReg), src0=vgpr(kReg), src1=vgpr(tReg), \ comment="6. offset in wave: lrOffset = bnOffset + lrKOffset")) + elif (kernel["MIInputPerThreadA"] == 8): + # 256b reads require additional offset, offset += int(serialId / 16) * 16 + module.add(VLShiftLeftAddU32(dst=vgpr(tReg), shiftHex=3, src0=vgpr(kReg), src1=vgpr(tReg), \ + comment="256b offset")) else: module.add(vectorStaticMultiplyAdd(vgpr(tReg), vgpr(kReg), strideK, vgpr(tReg), tmpSgprInfo, \ - "5. K offset: lrKOffset = kIdx * mStride(%u); 6. offset in wave: lrOffset = bnOffset + lrKOffset" % (strideK))) + "5. K offset: lrKOffset = kIdx * mStride(%u); 6. offset in wave: lrOffset = bnOffset + lrKOffset" % (strideK))) # wave offset if num1DWaves > 1: diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 18f8ef7018..1ed2fdf208 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -3994,7 +3994,6 @@ def lraFinalOffset(self, kernel, tP): with self.allocTmpSgpr(1) as tmpSgprInfo: module.add(vectorStaticMultiplyAdd(vgpr("LocalReadAddr%s"%tc), vgpr(rReg), kernel["LdsPad%s"%tc] * tP["bpeDS"], vgpr("LocalReadAddr%s"%tc), tmpSgprInfo, \ "Final Offset: padding %u per block %u" % (kernel["LdsPad%s"%tc] * tP["bpeDS"], kernel["LdsBlockSizePerPad%s"%tc]))) - # release resources self.vgprPool.checkIn(tmpVgpr) self.vgprPool.checkIn(wave_id) @@ -8262,7 +8261,6 @@ def globalReadBody(tP): # TODO: is it possible to load only hi16 when no in tail? (need to check INT8 too) datatype = kernel["ProblemType"]["DataType%s"%tc] if kernel["ConvertAfterDS"] else kernel["ProblemType"]["DataType"] isHigh16Bits = (datatype.isHalf() or datatype.isBFloat16()) and loopCnt%2==1 if not tP["isM"] else False - #Carson: global reads called here loadModule.add( self.chooseGlobalRead(kernel["BufferLoad"], \ bpl, destVgpr=destVgpr, \ addr0=vgpr(offsetVgpr), addr1=sgpr("Srd%s"%tc, 4), \ @@ -8270,7 +8268,8 @@ def globalReadBody(tP): glc=isGlc, slc=isSlc, nt=isNT, lds=isLds, \ hi16=isHigh16Bits , \ comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp))) - #instOffset += 16 #Carson: debug 256b reads + if tc == 'A': + instOffset += 16 # 256b reads require 16B offset if unrollMirrorWithSoffset: codeMod = Module("mirrorIdx%u"%loopCnt) @@ -9245,7 +9244,6 @@ def localWriteBody(tP): paramList[1] = paramList[1] - 65536 else: dstAddr=vgpr(lwa) - #carson: paramList[1] holds ds_write offset, but it is being split into two calls and 16 added to offset beyond this command somewhere ds = DSModifiers(na=1, offset=paramList[1]) writeInst = LocalWriteX(dstAddr=dstAddr, src=paramList[0], ds=ds, comment=comment) else: From 3246041e8dee6c48dfa1d9bc32b769d23fdf5841 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Tue, 24 Jun 2025 00:53:41 -0500 Subject: [PATCH 14/18] fixing broken tf32 16x16x16 kernels --- .../Tensile/Components/F32XEmulation.py | 52 +++++++++---------- tensilelite/Tensile/KernelWriterAssembly.py | 6 +-- .../Tensile/SolutionStructs/Solution.py | 2 - 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index 25dce2b35b..d700adf059 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -228,28 +228,28 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) - # if not kernel["EnableF32XEmulationLds"]: - # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) - # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) - # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) - # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+1"), src1=vgpr(tmp2))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) - # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh2, vgprMask="", vi=0)) - # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+2"), src1=vgpr(tmp1))) - # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) - # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) - # tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) - # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) - # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) - # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) - # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) - # tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) - # tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) - # tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) - # tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) - # tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) aStart = aStart + "+4" bStart = bStart + "+4" @@ -276,13 +276,13 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v vgprSize = width / 2 (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 1ed2fdf208..c84846bb64 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -8268,7 +8268,7 @@ def globalReadBody(tP): glc=isGlc, slc=isSlc, nt=isNT, lds=isLds, \ hi16=isHigh16Bits , \ comment="G -> Reg %u_%u_%u_%u"%(para, sPara, perp, sPerp))) - if tc == 'A': + if tc == 'A' and (kernel["MIInputPerThreadA"] == 8): instOffset += 16 # 256b reads require 16B offset if unrollMirrorWithSoffset: @@ -11807,8 +11807,8 @@ def bufferLoadImpl(soffset): dst2 = int(destVgpr + int(rpv//2)) dst = None if lds else vgpr(dst2, rpv//2) print("emulatedb2562 ", dst, addr0) - rv.add(SWaitCnt(vmcnt=0, comment="")) - #rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ + # rv.add(SWaitCnt(vmcnt=0, comment="")) + # rv.add(BufferLoadB128(dst=dst, vaddr=addr0, saddr=addr1, \ # soffset=soffset, mubuf=mubuf2, comment=comment)) return rv elif bpl==64 and not lds: diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 7acb9855a2..a08d5daf9a 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -2549,8 +2549,6 @@ def subCheckLdsBlockSizePerPad(tc, idx): state["NoLdsWriteCode"] = True # calculate ldsPad - result = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) - print("result: " + str(result)) state["LdsPadA"], state["LdsPadB"], state["LdsPadMetadata"] = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) if state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataType"].numBytes() == 32 and state["LdsPadA"] == 16 // state["ProblemType"]["DataType"].numBytes(): From 169714ac15ee2e3b5fcd987b2cdce61d07952a0a Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Wed, 25 Jun 2025 02:17:42 -0500 Subject: [PATCH 15/18] tf32 emulation optimization --- .../Tensile/Components/F32XEmulation.py | 48 +++++++++++++------ .../Tensile/Components/GlobalWriteBatch.py | 6 --- tensilelite/Tensile/Components/LocalRead.py | 5 -- tensilelite/Tensile/KernelWriterAssembly.py | 8 ++-- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index d700adf059..abcf01e9d6 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -177,11 +177,12 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(TextBlock("/*tf32 emulation*/\n")) aStart = re.search(r'v\[vgpr(.*?):', str(src1)).group(1) bStart = re.search(r'v\[vgpr(.*?):', str(src0)).group(1) - print("src0: {0} aStart: {1}".format(src0, aStart)) - print("src1: {0} bStart: {1}".format(src1, bStart)) + # print("src0: {0} aStart: {1}".format(src0, aStart)) + # print("src1: {0} bStart: {1}".format(src1, bStart)) tf32mod.add(TextBlock("/*f32 to 2 bfloat16 per input*/\n")) tf32mod.add(TextBlock("/*bf16AHigh*/\n")) numElementsPerIter = 4 + vgprSize = width / 2 #cvt0-1: Tmp #cvt2-5: AHigh @@ -194,7 +195,8 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v aLow = "Cvt+12" bLow = "Cvt+16" - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) for itr in range(int(width / numElementsPerIter)): #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) @@ -211,13 +213,21 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tmp1 = "Cvt+0" tmp2 = "Cvt+1" # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) if not kernel["EnableF32XEmulationLds"]: tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh1), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) - tf32mod.add(SNop(waitState=1, comment="1 wait s00tates for ds_read")) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh2), src0=vgpr(aStart + "+2"), src1=vgpr(aStart + "+3"))) tf32mod.add(TextBlock("/*bf16BHigh*/\n")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh1), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh2), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) + + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2)) if kernel["EnableF32XEmulationLds"]: None #tf32mod.add(SNop(waitState=1020, comment="1 wait states for ds_read")) @@ -239,6 +249,14 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh2, vgprMask="", vi=1)) tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + + tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) + # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SNop(waitState=64, comment="1 wait s00tates for ds_read")) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) @@ -272,22 +290,22 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) else: - tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) - vgprSize = width / 2 - (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) (src0, src1) = (vgpr(bLow,vgprSize), vgpr(aHigh,vgprSize)) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) - (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) - # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2)) + # tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + # (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) + # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2)) # tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + # tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + # (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) + # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SNop(waitState=64, comment="1 wait s00tates for ds_read")) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock(str("label_tf32_") + str(F32XEmulationMFMA.dbgCounter) + ":\n")) F32XEmulationMFMA.dbgCounter += 1 diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index 2de5820154..36f50bf3fe 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -394,7 +394,6 @@ def _prolog(self, module: Module): module.add(VMovB32(dst=vgpr(bufferOOB), src="BufferOOB")) else: bufferOOB = None - print("carsonGlobalWriteBatchWriter1") #when factorDim = 1 the bias's gwvw is alwasy be 1. factor_gwvw = 1 if self.factorDim else self.ss.cfg.gwvw for elementIdx, element in enumerate(self.batchElements): @@ -418,7 +417,6 @@ def _prolog(self, module: Module): sumIdxGSUSYNC = self.ss.elementSumIdx[elementIdx] module.add(addrCalc.emitAddressSetupCode(self.kernel, self.tPB, self.ss, self.tmpVgpr, self.tmpS01, self.edge, self.beta, self.atomic, elementIdx, addrDVgpr)) - print("carsonGlobalWriteBatchWriter2") if self.edge: module.add(addrCalc.edgeProtectCode(self.kernel, self.edge, self.beta, self.atomic, mask, self.tmpSgpr)) @@ -429,10 +427,8 @@ def _prolog(self, module: Module): module.add(addrCalc.emitLdChange(self.kernel, self.ss, 'C', self.edge, self.beta, mask, bufferOOB, (elementIdx == 0), self.tmpVgpr, self.tmpSgpr, addrCVgpr, self.addrC, 0)) if dataBeta not in loadedDataBeta: if self.kernel["GroupLoadStore"]: - print("carsonGlobalWriteBatchWriter3") loadInputCode.add(self.parentWriter.readInput(self.kernel, self.ss, 'C', self.kernel["ProblemType"]["DestDataType"], addrCalc, vc0, data, self.gwvw, addrCVgpr, self.tmpS01)) else: - print("carsonGlobalWriteBatchWriter3.2", elementIdx) module.add(self.parentWriter.readInput(self.kernel, self.ss, 'C', self.kernel["ProblemType"]["DestDataType"], addrCalc, vc0, data, self.gwvw, addrCVgpr, self.tmpS01)) loadedDataBeta[dataBeta] = ceil(self.kernel["ProblemType"]["DestDataType"].numBytes() * self.ss.cfg.gwvw / 16) self.loadsBetaIssued += ceil(self.kernel["ProblemType"]["DestDataType"].numBytes() * self.gwvw / 16) @@ -443,10 +439,8 @@ def _prolog(self, module: Module): if dataE not in loadedDataE: loadOffset = int((self.kernel["ProblemType"]["ComputeDataType"].numRegisters() - self.kernel["ProblemType"]["DataTypeE"].numRegisters()) * self.ss.cfg.gwvw) if self.kernel["GroupLoadStore"]: - print("carsonGlobalWriteBatchWriter4") loadInputCode.add(self.parentWriter.readInput(self.kernel, self.ss, 'E', self.kernel["ProblemType"]["DataTypeE"], addrCalc, vc0, dataE + loadOffset, self.gwvw, addrEVgpr, self.tmpS01)) else: - print("carsonGlobalWriteBatchWriter4.2") module.add(self.parentWriter.readInput(self.kernel, self.ss, 'E', self.kernel["ProblemType"]["DataTypeE"], addrCalc, vc0, dataE + loadOffset, self.gwvw, addrEVgpr, self.tmpS01)) loadedDataE[dataE] = ceil(self.kernel["ProblemType"]["DataTypeE"].numBytes() * self.ss.cfg.gwvw / 16) self.loadsEIssued += ceil(self.kernel["ProblemType"]["DataTypeE"].numBytes() * self.gwvw / 16) diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index 0e2c625e9a..f49901c421 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -216,7 +216,6 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): numSplitMetadata = max(ceil((blockWidth * 4) // (kernel["MIInputPerThread%s"%tc] * tP["bpeDS"])) - 1, 0) if tP["isM"] else 0 valufIdx = 0 if enableLDSTr: - print("WEEEE1") numberMTilesPerWave = kernel["MIWaveTile"][tile01] highBits = 0 for tIdx in range(0, numberMTilesPerWave): @@ -238,7 +237,6 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): ds = DSModifiers(na=1, offset=paramList[1]) localReadCode.add(LocalReadX(dst=destVgpr, src=vgpr("LocalReadAddr%s"%tc), ds=ds, comment=comment)) else: - print("WEEEE2") for vIdx in range(0, numVectorsPerTile): for eIdx in range(0, numReadsPerVector): valuiIdx = int(valufIdx) @@ -547,13 +545,10 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): incOffset = rIdx * numElementPerRead * UnrollStride + incOffset offset_val = (incOffset + offset_val + tP["localReadOffset"]) * tP["bpeDS"] else: - print("offsetval: {0}, {1}, {2}", offset_val, numElementPerRead, UnrollStride) offset_val = (rIdx * numElementPerRead * UnrollStride + offset_val + tP["localReadOffset"]) * tP["bpeDS"] - print("offsetval2: ", offset_val) if (kernel["LdsBlockSizePerPad%s"%tc] != 0) and (kernel["LdsPad%s"%tc] != 0): offset_val = offset_val + (offset_val // kernel["LdsBlockSizePerPad%s"%tc]) * kernel["LdsPad%s"%tc] * tP["bpeDS"] - print("offsetval3: ", offset_val, kernel["LdsBlockSizePerPad%s"%tc], kernel["LdsPad%s"%tc], tP["bpeDS"]) offset_val = offset_val + tP["localReadSwapByteOffset"] # offset_val = offset_val * 4 #Carson: Debug if (kernel["DirectToLds%s" % tc] and \ diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index c84846bb64..d9fb148339 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -11841,10 +11841,10 @@ def bufferLoadImpl(soffset): else: assert 0, "%s\nchooseGlobalRead: bad bpl %u"%(self.states.kernelName,bpl) - global dbgCounter - rv.add(SWaitCnt(lgkmcnt=0, comment="")) - rv.add(TextBlock(str("label_gr_") + str(dbgCounter) + ":\n")) - dbgCounter += 1 + # global dbgCounter + # rv.add(SWaitCnt(lgkmcnt=0, comment="")) + # rv.add(TextBlock(str("label_gr_") + str(dbgCounter) + ":\n")) + # dbgCounter += 1 # buffer_load offset field is 12-bit. # if offset >= 4096, use soffset instead From 01acd5621a4871d362a1caaccbc881a4054831f7 Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Thu, 26 Jun 2025 17:58:28 -0500 Subject: [PATCH 16/18] tf32 fix for lds --- .../Tensile/Components/F32XEmulation.py | 65 +++++++++++++++---- .../Tensile/SolutionStructs/Solution.py | 2 +- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index abcf01e9d6..c3f8dc139c 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -125,7 +125,7 @@ def __call__(self, dstStart): tf32mod = Module() # Carson: textblock here (or in localread) is causing python issues. rocisa ambiguity issue? tf32mod.add(TextBlock("/*TF32 Emulation read lds*/\n")) - tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) + # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) #tf32mod.add(SNop(waitState=27, comment="wait for ds_read")) tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 @@ -162,7 +162,7 @@ def __call__(self, dstStart): # 3: [2low, 3low] # - tf32mod.add(TextBlock(str("label_tf32Read_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) + tf32mod.add(TextBlock(str("label_tf32Read_end_") + str(F32XEmulationCvtLocalRead.dbgCounter) + ":\n")) F32XEmulationCvtLocalRead.dbgCounter += 1 return tf32mod @@ -184,6 +184,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v numElementsPerIter = 4 vgprSize = width / 2 + # Cvt register layout: #cvt0-1: Tmp #cvt2-5: AHigh #cvt6-9: Bhigh @@ -222,12 +223,30 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh1), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh2), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + if kernel["EnableF32XEmulationLds"]: + (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + else: (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aHigh,vgprSize)) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2)) + + for itr in range(int(width / numElementsPerIter)): + offset = "+" + str(itr * 2) + aHigh1 = aHigh + offset + aHigh2 = aHigh + "+1" + offset + aLow1 = aLow + offset + aLow2 = aLow + "+1" + offset + bHigh1 = bHigh + offset + bHigh2 = bHigh + "+1" + offset + bLow1 = bLow + offset + bLow2 = bLow + "+1" + offset + tmp1 = "Cvt+0" + tmp2 = "Cvt+1" if kernel["EnableF32XEmulationLds"]: None #tf32mod.add(SNop(waitState=1020, comment="1 wait states for ds_read")) @@ -250,12 +269,30 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(aStart+"+3"), src1=vgpr(tmp2))) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) - tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + if kernel["EnableF32XEmulationLds"]: + (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + else: (src0, src1) = (vgpr(bHigh,vgprSize), vgpr(aLow,vgprSize)) # # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) # tf32mod.add(SNop(waitState=64, comment="1 wait s00tates for ds_read")) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + + for itr in range(int(width / numElementsPerIter)): + offset = "+" + str(itr * 2) + aHigh1 = aHigh + offset + aHigh2 = aHigh + "+1" + offset + aLow1 = aLow + offset + aLow2 = aLow + "+1" + offset + bHigh1 = bHigh + offset + bHigh2 = bHigh + "+1" + offset + bLow1 = bLow + offset + bLow2 = bLow + "+1" + offset + tmp1 = "Cvt+0" + tmp2 = "Cvt+1" tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) @@ -274,20 +311,20 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v #todo: working impl using in situ cvt cmd. lds currently some kernels are failing if kernel["EnableF32XEmulationLds"]: - tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) + # (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) (src0, src1) = (vgpr("Cvt+6",2), vgpr(aStart,2)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) - tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) + # tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) - tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ - acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) + # (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) + # tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ + # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) else: tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index a08d5daf9a..f24b8d213b 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -891,7 +891,7 @@ def assignDerivedParameters( # the keys "EnableF32XdlMathOp" and "F32XdlMathOp". state["EnableF32XdlMathOp"] = False state["UseF32XEmulation"] = False #enable emulation for missing hardware support - state["EnableF32XEmulationLds"] = False + state["EnableF32XEmulationLds"] = True #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ From d08bfbc2bc5a16491735e27b91d3a7c5e50916aa Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Mon, 30 Jun 2025 14:41:22 -0500 Subject: [PATCH 17/18] tf32 performance optimizations. k32 MI working with 256 tile sizes --- .../Tensile/Components/F32XEmulation.py | 40 ++++++++++--------- tensilelite/Tensile/Components/LocalRead.py | 2 +- tensilelite/Tensile/KernelWriter.py | 2 + tensilelite/Tensile/KernelWriterAssembly.py | 6 +-- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tensilelite/Tensile/Components/F32XEmulation.py b/tensilelite/Tensile/Components/F32XEmulation.py index c3f8dc139c..b63251a32f 100644 --- a/tensilelite/Tensile/Components/F32XEmulation.py +++ b/tensilelite/Tensile/Components/F32XEmulation.py @@ -216,17 +216,18 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(TextBlock("/*bf16AHigh*/\n")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh1), src0=vgpr(aStart), src1=vgpr(aStart + "+1"))) tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(VCvtPkF32toBF16(dst=vgpr(aHigh2), src0=vgpr(aStart + "+2"), src1=vgpr(aStart + "+3"))) - tf32mod.add(TextBlock("/*bf16BHigh*/\n")) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh1), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh2), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) + tf32mod.add(TextBlock("/*bf16BHigh*/\n")) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh1), src0=vgpr(bStart), src1=vgpr(bStart + "+1"))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bHigh2), src0=vgpr(bStart + "+2"), src1=vgpr(bStart + "+3"))) - tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) + # tf32mod.add(SNop(waitState=1, comment="1 wait states for ds_read")) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) if kernel["EnableF32XEmulationLds"]: - (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart,2)) + (src0, src1) = (vgpr(bStart,vgprSize), vgpr(aStart,vgprSize)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) else: @@ -256,8 +257,8 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v #tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) #tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(TextBlock("/*bf16ALow = A - float32(bf16AHigh)*/\n")) tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=aHigh1, vgprMask="", vi=0)) tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(aStart+"+0"), src1=vgpr(tmp1))) tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=aHigh1, vgprMask="", vi=1)) @@ -271,7 +272,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tf32mod.add(TextBlock("/*acc = bf16ALow * bf16BHigh*/\n")) if kernel["EnableF32XEmulationLds"]: - (src0, src1) = (vgpr("Cvt+2",2), vgpr(aStart + "+2",2)) + (src0, src1) = (vgpr(bStart,vgprSize), vgpr(aStart + "+" + str(vgprSize),vgprSize)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) else: @@ -294,17 +295,18 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v tmp1 = "Cvt+0" tmp2 = "Cvt+1" - tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) - tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) - tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) - tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) - tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) - tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) - tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) - tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) + if not kernel["EnableF32XEmulationLds"]: + tf32mod.add(TextBlock("/*bf16BLow = B - float32(bf16BHigh)*/\n")) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh1, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+0"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh1, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+1"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow1), src0=vgpr(tmp1), src1=vgpr(tmp2))) + tf32mod.add(VCvtBF16toFP32(dst=tmp1, src=bHigh2, vgprMask="", vi=0)) + tf32mod.add(VSubF32(dst=vgpr(tmp1), src0=vgpr(bStart+"+2"), src1=vgpr(tmp1))) + tf32mod.add(VCvtBF16toFP32(dst=tmp2, src=bHigh2, vgprMask="", vi=1)) + tf32mod.add(VSubF32(dst=vgpr(tmp2), src0=vgpr(bStart+"+3"), src1=vgpr(tmp2))) + tf32mod.add(VCvtPkF32toBF16(dst=vgpr(bLow2), src0=vgpr(tmp1), src1=vgpr(tmp2))) aStart = aStart + "+4" bStart = bStart + "+4" @@ -317,7 +319,7 @@ def __call__(self, kernel, acc, acc2, src0, src1, miInInstType, miOutInstType, v # acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BLow*/\n")) # tf32mod.add(SWaitCnt(lgkmcnt=0, comment="wait for lds read")) - (src0, src1) = (vgpr("Cvt+6",2), vgpr(aStart,2)) + (src0, src1) = (vgpr(bStart + "+" + str(vgprSize),vgprSize), vgpr(aStart,vgprSize)) tf32mod.add(MFMAInstruction(instType=InstType.INST_BF16, accType=miOutInstType, variant=variant, mfma1k=mfma_1k, \ acc=acc, a=src0, b=src1, acc2=acc2, neg=neg_flag)) # tf32mod.add(TextBlock("/*acc += bf16AHigh * bf16BHigh*/\n")) diff --git a/tensilelite/Tensile/Components/LocalRead.py b/tensilelite/Tensile/Components/LocalRead.py index f49901c421..217809871c 100644 --- a/tensilelite/Tensile/Components/LocalRead.py +++ b/tensilelite/Tensile/Components/LocalRead.py @@ -624,7 +624,7 @@ def __call__(self, writer, kernel, bufferIdx, iui, epsi, tP): elif kernel["ProblemType"]["DataType"].isSingle(): localReadCode.add(writer.assert_eq( dbgVgpr, 1.0) ) - if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"] and tP["isA"] and tc == "A": + if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: tf32mod = F32XEmulationCvtLocalRead() dstStart = "Valu%s_X%u_I%u+%u"%(tc, bufferIdx, iui, baseValuiIdx) localReadCode.add(tf32mod(dstStart)) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 5ba5d3a089..83397a1051 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1084,6 +1084,8 @@ def checkLocalReadFIFOFull(currentMFMA, fifo, lrItems, numLR, numLREven): insertedPackM = 0 def hasDependency(lr: DSLoadInstruction, inst: Instruction) -> bool: + if not (hasattr(lr, 'dst') and callable(lr.dst)): + return False lrDataReg = lr.dst if isinstance(inst, MFMAInstruction): diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index d9fb148339..8027a6f305 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -9251,9 +9251,9 @@ def localWriteBody(tP): writeInst = LocalWriteX(dstAddr=vgpr(lwa), src0=paramList[0], src1=paramList[1], ds=ds, comment=comment) if kernel["UseF32XEmulation"] and kernel["EnableF32XEmulationLds"]: vgrStr = str(destVgprPrefix + "+%u"%(g2lIdx + eccOffset)) - if "A" in vgrStr: - emulationLocalWrite = F32XEmulationCvtLocalWrite() - localWriteCode.add(emulationLocalWrite(vgrStr)) + # if "A" in vgrStr: + emulationLocalWrite = F32XEmulationCvtLocalWrite() + localWriteCode.add(emulationLocalWrite(vgrStr)) if self.do["LocalWriteCVT"]: localWriteCode.add(localWriteCVTCode) if self.do["LocalWrite%s"%tc]: From 6e7a9be6763920a5a5006577e5a6e252e8d5823d Mon Sep 17 00:00:00 2001 From: carsonbrownlee Date: Mon, 30 Jun 2025 18:32:31 -0500 Subject: [PATCH 18/18] tf32 yaml for high compute intensity --- ...gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml | 156 +++++++++--------- 1 file changed, 79 insertions(+), 77 deletions(-) diff --git a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml index 139de69e69..dfa079ec90 100644 --- a/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml +++ b/library/src/amd_detail/rocblaslt/src/Tensile/Logic/asm_full/gfx950/GridBased/gfx950_Cijk_Alik_Bljk_S_MX_B_UserArgs.yaml @@ -89,36 +89,38 @@ AssertSummationElementMultiple: 1 AssignedDerivedParameters: true AssignedProblemIndependentDerivedParameters: true - BaseName: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT64x16x16_MI16x1J7V_jobmP44OFztrsbXl-d3dSIK0CEdTi8QZcUodlVY= + BaseName: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT256x256x32_MI16GZB8tvR2VQN0ZSch4g4iOm8t5mZh5aOHgP_1vzk34Bo= BufferLoad: true BufferStore: true CUCount: null + CUOccupancy: -1 ClusterLocalRead: 0 CodeObjectVersion: '4' ConvertAfterDS: false CustomKernelName: '' DebugStreamK: 0 - DepthU: 16 - DirectToLds: false + DepthU: 32 + DirectToLds: 0 DirectToLdsA: false DirectToLdsB: false DirectToVgprA: false DirectToVgprB: false DirectToVgprSparseMetadata: false EdgeType: ShiftPtr - EnableF32XEmulationLds: false + EnableF32XEmulationLds: true EnableF32XdlMathOp: true EnableMatrixInstruction: true - ExpandPointerSwap: true + ExpandPointerSwap: 0 + ExpertSchedulingMode: 0 ForceDisableShadowInit: false GlobalReadPerMfma: 1 GlobalReadVectorWidthA: 4 - GlobalReadVectorWidthB: 1 + GlobalReadVectorWidthB: 4 GlobalSplitU: 1 GlobalSplitUAlgorithm: MultipleBuffer GlobalSplitUCoalesced: false GlobalSplitUWorkGroupMappingRoundRobin: false - GlobalWriteVectorWidth: 1 + GlobalWriteVectorWidth: 4 GroupLoadStore: false GuaranteeNoPartialA: true GuaranteeNoPartialB: true @@ -130,34 +132,34 @@ SupportUserGSU: true, UseUniversalArgs: true} Kernel: true KernelLanguage: Assembly - KernelNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_K1_MIWT1_1 + KernelNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT256x256x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA4_GRVWB4_GSUAMB_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB512_LBSPPM0_LPA8_LPB8_LPM0_LRVW4_LWPMn1_MIAV0_MIWT8_8_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SPO0_SRVW0_SSO0_SVW4_SK0_SKXCCM0_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA4_VWB4_WSGRA0_WSGRB0_WS64_WG32_8_1 LDSTrInst: false - LSCA: 16 - LSCB: 16 - LSPA: 64 - LSPB: 16 - LVCA: 4 - LVCB: 16 - LVPA: 16 - LVPB: 16 - LdsBlockSizePerPadA: 128 - LdsBlockSizePerPadB: 128 + LSCA: 32 + LSCB: 32 + LSPA: 32 + LSPB: 32 + LVCA: 8 + LVCB: 8 + LVPA: 8 + LVPB: 8 + LdsBlockSizePerPadA: 512 + LdsBlockSizePerPadB: 512 LdsBlockSizePerPadMetadata: 0 - LdsBytesNoAmax: 14592 + LdsBytesNoAmax: 139264 LdsInitCVgprs: false - LdsNumBytes: 14592 - LdsNumElementsAlignedA: 5120 - LdsNumElementsAlignedB: 1280 + LdsNumBytes: 139264 + LdsNumElementsAlignedA: 34816 + LdsNumElementsAlignedB: 34816 LdsNumElementsAlignedMetadata: 0 LdsOffsetA: 0 - LdsOffsetA_Blk: 8192 - LdsOffsetB: 5120 - LdsOffsetB_Blk: 13312 + LdsOffsetA_Blk: 69632 + LdsOffsetB: 34816 + LdsOffsetB_Blk: 104448 LdsOffsetBias: 0 LdsOffsetBiasGSU: 0 LdsOffsetBiasNonGSU: 0 - LdsOffsetMetadata: 5120 - LdsOffsetMetadata_Blk: 13312 + LdsOffsetMetadata: 34816 + LdsOffsetMetadata_Blk: 104448 LdsPadA: 8 LdsPadB: 8 LdsPadMetadata: 0 @@ -168,33 +170,34 @@ LocalWriteUseSgprA: false LocalWriteUseSgprB: false LoopIters: 1 - LoopUnroll: 16 + LoopUnroll: 32 MFMA_BF16_1K: false MIArchVgpr: false - MIBlock: [16, 16, 16, 1, 1, 1] - MIInputPerThread: 4 - MIInputPerThreadA: 4 - MIInputPerThreadB: 4 - MIInputPerThreadMetadata: 4 + MIBlock: [16, 16, 32, 1, 1, 1] + MIInputPerThread: 8 + MIInputPerThreadA: 8 + MIInputPerThreadB: 8 + MIInputPerThreadMetadata: 8 MIOutputVectorWidth: 4 MIRegPerOut: 1 - MIWaveGroup: [4, 1] - MIWaveTile: [1, 1] - MIWaveTileA: 1 - MIWaveTileB: 1 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 8] + MIWaveTileA: 8 + MIWaveTileB: 8 MIWaveTileMetadata: 0 - MacroTile0: 64 - MacroTile1: 16 - MacroTileA: 64 - MacroTileB: 16 + MacroTile0: 256 + MacroTile1: 256 + MacroTileA: 256 + MacroTileB: 256 MagicDivAlg: 2 + MathClocksUnrolledLoop: 0 MatrixInstB: 1 MatrixInstBM: 1 MatrixInstBN: 1 - MatrixInstK: 16 + MatrixInstK: 32 MatrixInstM: 16 MatrixInstN: 16 - MatrixInstruction: [16, 16, 16, 1] + MatrixInstruction: [16, 16, 32, 1] MaxOccupancy: 40 MbskPrefetchOpt: 0 NoLdsWriteCode: false @@ -209,14 +212,14 @@ NonTemporalMetadata: 0 NonTemporalWS: 0 NumElementsPerBatchStore: 0 - NumElementsPerThread: 4 - NumGlobalWriteVectorsPerThread: 4 - NumLoadsA: 1 - NumLoadsB: 1 + NumElementsPerThread: 256 + NumGlobalWriteVectorsPerThread: 64 + NumLoadsA: 8 + NumLoadsB: 8 NumLoadsCoalescedA: 1 NumLoadsCoalescedB: 1 - NumLoadsPerpendicularA: 1 - NumLoadsPerpendicularB: 1 + NumLoadsPerpendicularA: 8 + NumLoadsPerpendicularB: 8 NumThreads: 256 NumWaveSplitK: 1 OptNoLoadLoop: 1 @@ -224,9 +227,9 @@ PackedC0IndicesX: [0] PackedC1IdxChars: [J] PackedC1IndicesX: [1] - PrefetchGlobalRead: 1 + PrefetchGlobalRead: 2 PrefetchLocalRead: 0 - PreloadKernArgs: 0 + PreloadKernArgs: true ProblemType: Activation: false ActivationComputeDataType: 0 @@ -305,31 +308,32 @@ UseScaleAlphaVec: 0 UseScaleCD: false ScheduleGlobalRead: 1 - ScheduleIterAlg: 1 + ScheduleIterAlg: 3 ScheduleLocalWrite: 1 SolutionIndex: 0 - SolutionNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT64x16x16_MI16x16x1_SN_GSU1_GSUC0_GSUWGMRR0_K1_MIWT1_1_SU32_SUM0_SUS256_WGM8_WGMXCC1_WGMXCCGn1 + SolutionNameMin: Cijk_Alik_Bljk_S_MX_B_UserArgs_MT256x256x32_MI16x16x1_SN_LDSB0_AFC0_AFEM1_AFEM1_ASEM1_CLR0_CADS0_DTVA0_DTVB0_EPS0_FDSI0_GRPM1_GRVWA4_GRVWB4_GSU1_GSUAMB_GSUC0_GSUWGMRR0_GLS0_ISA950_IU1_K1_LBSPPA512_LBSPPB512_LBSPPM0_LPA8_LPB8_LPM0_LRVW4_LWPMn1_MIAV0_MIWT8_8_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB1_ONLL1_PGR2_PLR0_PKA1_SIA3_SS1_SU0_SUM0_SUS128_SPO0_SRVW0_SSO0_SVW4_SK0_SKXCCM0_TLDS1_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA4_VWB4_WSGRA0_WSGRB0_WS64_WG32_8_1_WGM8_WGMXCC1_WGMXCCGn1 SourceSwap: 1 - StaggerU: 32 + StaggerU: 0 StaggerUMapping: 0 - StaggerUStride: 256 + StaggerUStride: 128 StorePriorityOpt: false StoreRemapVectorWidth: 0 + StoreSwapAddr: true StoreSyncOpt: 0 - StoreVectorWidth: 1 + StoreVectorWidth: 4 StreamK: 0 StreamKAtomic: 0 StreamKXCCMapping: 0 - SubGroup0: 16 - SubGroup1: 16 - SubGroupA: 16 - SubGroupB: 16 + SubGroup0: 8 + SubGroup1: 32 + SubGroupA: 8 + SubGroupB: 32 SuppressNoLoadLoop: false ThreadTile: [1, 1] - ThreadTile0: 4 - ThreadTile1: 1 - ThreadTileA: 4 - ThreadTileB: 1 + ThreadTile0: 32 + ThreadTile1: 8 + ThreadTileA: 32 + ThreadTileB: 8 TransposeLDS: 1 TransposeLDSMetadata: true ULSGRODoubleG2L: 0 @@ -339,34 +343,34 @@ UnrollMajorLDSMetadata: true Use64bShadowLimit: 1 UseDotInstruction: false - UseF32XEmulation: false + UseF32XEmulation: true UseInstOffsetForGRO: 0 UseSgprForGRO: -1 Valid: true VectorStore: -1 - VectorWidthA: 1 - VectorWidthB: 1 + VectorWidthA: 4 + VectorWidthB: 4 WaveSeparateGlobalReadA: 0 WaveSeparateGlobalReadB: 0 WaveSeparateGlobalReadMetadata: 0 WaveSplitK: false WavefrontSize: 64 - WorkGroup: [64, 4, 1] + WorkGroup: [32, 8, 1] WorkGroupMapping: 8 WorkGroupMappingXCC: 1 WorkGroupMappingXCCGroup: -1 WorkGroupReduction: false WorkspaceCheck: [4, 0, 1] - _DepthU: 16 - _DepthUA: 16 - _DepthUB: 16 - _DepthUMetadata: 16 + _DepthU: 32 + _DepthUA: 32 + _DepthUB: 32 + _DepthUMetadata: 32 _GlobalAccumulation: MultipleBuffer _UseSgprForGRO: 1 _VectorStore: 1 _WorkspaceSizePerElemBias: 0 _WorkspaceSizePerElemC: 4 - _staggerStrideShift: 2 + _staggerStrideShift: 0 enableLDSTrA: false enableLDSTrB: false reorderGRInstForDTVA: false @@ -374,10 +378,8 @@ tailLoopOptA: false tailLoopOptB: false - [2, 3, 0, 1] -- - - [128, 128, 1, 128, 128, 128, 128, 128] - - [0, 0.18] - - - [2048, 2048, 1, 256, 2048, 2048, 256, 256] - - [0, 15.14] +- - - [4096, 4096, 1, 16384, 4096, 4096, 16384, 16384] + - [0, 153.51] - null - null - DeviceEfficiency