From 284bcbae13a565795520305a008660f1249d2b21 Mon Sep 17 00:00:00 2001 From: pengxu Date: Tue, 4 Aug 2026 10:51:46 +0800 Subject: [PATCH] loongarch: add small matrix kernel of sgemm_lasx/lsx and dgemm_lsx --- kernel/loongarch64/KERNEL.LA264 | 21 + kernel/loongarch64/KERNEL.LA464 | 10 + .../loongarch64/dgemm_small_kernel_nn_lsx.S | 738 ++++++++++++++++ .../loongarch64/dgemm_small_kernel_nt_lsx.S | 468 +++++++++++ .../loongarch64/dgemm_small_kernel_tn_lsx.S | 575 +++++++++++++ .../loongarch64/dgemm_small_kernel_tt_lsx.S | 559 ++++++++++++ .../loongarch64/dgemm_small_matrix_permit.c | 12 + .../loongarch64/sgemm_small_kernel_nn_lasx.S | 566 +++++++++++++ .../loongarch64/sgemm_small_kernel_nn_lsx.S | 795 ++++++++++++++++++ .../loongarch64/sgemm_small_kernel_nt_lasx.S | 499 +++++++++++ .../loongarch64/sgemm_small_kernel_nt_lsx.S | 614 ++++++++++++++ .../loongarch64/sgemm_small_kernel_tn_lasx.S | 764 +++++++++++++++++ .../loongarch64/sgemm_small_kernel_tn_lsx.S | 733 ++++++++++++++++ .../loongarch64/sgemm_small_kernel_tt_lasx.S | 719 ++++++++++++++++ .../loongarch64/sgemm_small_kernel_tt_lsx.S | 680 +++++++++++++++ .../loongarch64/sgemm_small_matrix_permit.c | 56 ++ 16 files changed, 7809 insertions(+) create mode 100644 kernel/loongarch64/dgemm_small_kernel_nn_lsx.S create mode 100644 kernel/loongarch64/dgemm_small_kernel_nt_lsx.S create mode 100644 kernel/loongarch64/dgemm_small_kernel_tn_lsx.S create mode 100644 kernel/loongarch64/dgemm_small_kernel_tt_lsx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_nn_lasx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_nn_lsx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_nt_lasx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_nt_lsx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_tn_lasx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_tn_lsx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_tt_lasx.S create mode 100644 kernel/loongarch64/sgemm_small_kernel_tt_lsx.S create mode 100644 kernel/loongarch64/sgemm_small_matrix_permit.c diff --git a/kernel/loongarch64/KERNEL.LA264 b/kernel/loongarch64/KERNEL.LA264 index 18f18af738..4f998fdb6e 100644 --- a/kernel/loongarch64/KERNEL.LA264 +++ b/kernel/loongarch64/KERNEL.LA264 @@ -143,4 +143,25 @@ ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +SGEMM_SMALL_M_PERMIT = sgemm_small_matrix_permit.c +SGEMM_SMALL_K_NN = sgemm_small_kernel_nn_lsx.S +SGEMM_SMALL_K_B0_NN = sgemm_small_kernel_nn_lsx.S +SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_lsx.S +SGEMM_SMALL_K_B0_NT = sgemm_small_kernel_nt_lsx.S +SGEMM_SMALL_K_TN = sgemm_small_kernel_tn_lsx.S +SGEMM_SMALL_K_B0_TN = sgemm_small_kernel_tn_lsx.S +SGEMM_SMALL_K_TT = sgemm_small_kernel_tt_lsx.S +SGEMM_SMALL_K_B0_TT = sgemm_small_kernel_tt_lsx.S + +DGEMM_SMALL_M_PERMIT = dgemm_small_matrix_permit.c +DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_lsx.S +DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_lsx.S +DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_lsx.S +DGEMM_SMALL_K_B0_NT = dgemm_small_kernel_nt_lsx.S +DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_lsx.S +DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_lsx.S +DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_lsx.S +DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_lsx.S + endif diff --git a/kernel/loongarch64/KERNEL.LA464 b/kernel/loongarch64/KERNEL.LA464 index ca8c4d3884..bf2d0a5385 100644 --- a/kernel/loongarch64/KERNEL.LA464 +++ b/kernel/loongarch64/KERNEL.LA464 @@ -163,6 +163,16 @@ STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c +SGEMM_SMALL_M_PERMIT = sgemm_small_matrix_permit.c +SGEMM_SMALL_K_NN = sgemm_small_kernel_nn_lasx.S +SGEMM_SMALL_K_B0_NN = sgemm_small_kernel_nn_lasx.S +SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_lasx.S +SGEMM_SMALL_K_B0_NT = sgemm_small_kernel_nt_lasx.S +SGEMM_SMALL_K_TN = sgemm_small_kernel_tn_lasx.S +SGEMM_SMALL_K_B0_TN = sgemm_small_kernel_tn_lasx.S +SGEMM_SMALL_K_TT = sgemm_small_kernel_tt_lasx.S +SGEMM_SMALL_K_B0_TT = sgemm_small_kernel_tt_lasx.S + DGEMM_SMALL_M_PERMIT = dgemm_small_matrix_permit.c DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_lasx.S DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_lasx.S diff --git a/kernel/loongarch64/dgemm_small_kernel_nn_lsx.S b/kernel/loongarch64/dgemm_small_kernel_nn_lsx.S new file mode 100644 index 0000000000..d237fd0f8a --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_nn_lsx.S @@ -0,0 +1,738 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K8 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro DGEMM_SMALL_KERNEL_NN_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M\M\()_N3 +.L_M\M\()_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M\M\()_N4_END +.L_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vldrepl.d Z0, X0, 0x00 + vldrepl.d Z1, B1, 0x00 + vldrepl.d Z2, B2, 0x00 + vldrepl.d Z3, B3, 0x00 + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S0, Z1, D1 + vfmadd.d D2, S0, Z2, D2 + vfmadd.d D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N4_K1 +.L_M\M\()_N4_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA + vfmul.d D2, D2, VALPHA + vfmul.d D3, D3, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2 + vld S0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2 + fld.d FS0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00 + vst V1, C1, 0x00 + vst V2, C2, 0x00 + vst V3, C3, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 + fst.d F2, C2, 0x00 + fst.d F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M\M\()_N4 +.L_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_M\M\()_N1 +.L_M\M\()_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_M\M\()_N2_END +.L_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vldrepl.d Z0, X0, 0x00 + vldrepl.d Z1, B1, 0x00 + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N2_K1 +.L_M\M\()_N2_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00 + vst V1, C1, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + bge ZERO, K, .L_M\M\()_N1_END +.L_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N1_K1 +.L_M\M\()_N1_END: + vfmul.d D0, D0, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00 +.endif +.L_M\M\()_END: +.if \M == 2 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 1 + PTR_ADDI A, A, 0x08 + PTR_ADDI C, C, 0x08 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + + vreplvei.d VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.d VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_SLLI K8, K, 3 + PTR_SRAI M8, M, 3 // M >> 3 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + vxor.v D8, D8, D8 + vxor.v D9, D9, D9 + vxor.v D10, D10, D10 + vxor.v D11, D11, D11 + vxor.v D12, D12, D12 + vxor.v D13, D13, D13 + vxor.v D14, D14, D14 + vxor.v D15, D15, D15 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.d Z0, X0, 0x00 + vldrepl.d Z1, B1, 0x00 + vldrepl.d Z2, B2, 0x00 + vldrepl.d Z3, B3, 0x00 + + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S2, Z0, D2 + vfmadd.d D3, S3, Z0, D3 + + vfmadd.d D4, S0, Z1, D4 + vfmadd.d D5, S1, Z1, D5 + vfmadd.d D6, S2, Z1, D6 + vfmadd.d D7, S3, Z1, D7 + + vfmadd.d D8, S0, Z2, D8 + vfmadd.d D9, S1, Z2, D9 + vfmadd.d D10, S2, Z2, D10 + vfmadd.d D11, S3, Z2, D11 + + vfmadd.d D12, S0, Z3, D12 + vfmadd.d D13, S1, Z3, D13 + vfmadd.d D14, S2, Z3, D14 + vfmadd.d D15, S3, Z3, D15 + + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA + vfmul.d D2, D2, VALPHA + vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA + vfmul.d D5, D5, VALPHA + vfmul.d D6, D6, VALPHA + vfmul.d D7, D7, VALPHA + vfmul.d D8, D8, VALPHA + vfmul.d D9, D9, VALPHA + vfmul.d D10, D10, VALPHA + vfmul.d D11, D11, VALPHA + vfmul.d D12, D12, VALPHA + vfmul.d D13, D13, VALPHA + vfmul.d D14, D14, VALPHA + vfmul.d D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vld S2, C0, 0x20 + vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0 + vfmadd.d D1, S1, VBETA, D1 + vfmadd.d D2, S2, VBETA, D2 + vfmadd.d D3, S3, VBETA, D3 + + vld S0, C1, 0x00 + vld S1, C1, 0x10 + vld S2, C1, 0x20 + vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4 + vfmadd.d D5, S1, VBETA, D5 + vfmadd.d D6, S2, VBETA, D6 + vfmadd.d D7, S3, VBETA, D7 + + vld S0, C2, 0x00 + vld S1, C2, 0x10 + vld S2, C2, 0x20 + vld S3, C2, 0x30 + vfmadd.d D8, S0, VBETA, D8 + vfmadd.d D9, S1, VBETA, D9 + vfmadd.d D10, S2, VBETA, D10 + vfmadd.d D11, S3, VBETA, D11 + + vld S0, C3, 0x00 + vld S1, C3, 0x10 + vld S2, C3, 0x20 + vld S3, C3, 0x30 + vfmadd.d D12, S0, VBETA, D12 + vfmadd.d D13, S1, VBETA, D13 + vfmadd.d D14, S2, VBETA, D14 + vfmadd.d D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C0, 0x20 + vst D3, C0, 0x30 + vst D4, C1, 0x00 + vst D5, C1, 0x10 + vst D6, C1, 0x20 + vst D7, C1, 0x30 + vst D8, C2, 0x00 + vst D9, C2, 0x10 + vst D10, C2, 0x20 + vst D11, C2, 0x30 + vst D12, C3, 0x00 + vst D13, C3, 0x10 + vst D14, C3, 0x20 + vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + move K1, K + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.d Z0, X0, 0x00 + vldrepl.d Z1, B1, 0x00 + + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S2, Z0, D2 + vfmadd.d D3, S3, Z0, D3 + + vfmadd.d D4, S0, Z1, D4 + vfmadd.d D5, S1, Z1, D5 + vfmadd.d D6, S2, Z1, D6 + vfmadd.d D7, S3, Z1, D7 + + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA + vfmul.d D2, D2, VALPHA + vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA + vfmul.d D5, D5, VALPHA + vfmul.d D6, D6, VALPHA + vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vld S2, C0, 0x20 + vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0 + vfmadd.d D1, S1, VBETA, D1 + vfmadd.d D2, S2, VBETA, D2 + vfmadd.d D3, S3, VBETA, D3 + + vld S0, C1, 0x00 + vld S1, C1, 0x10 + vld S2, C1, 0x20 + vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4 + vfmadd.d D5, S1, VBETA, D5 + vfmadd.d D6, S2, VBETA, D6 + vfmadd.d D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C0, 0x20 + vst D3, C0, 0x30 + vst D4, C1, 0x00 + vst D5, C1, 0x10 + vst D6, C1, 0x20 + vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S2, Z0, D2 + vfmadd.d D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA + vfmul.d D2, D2, VALPHA + vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vld S2, C0, 0x20 + vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0 + vfmadd.d D1, S1, VBETA, D1 + vfmadd.d D2, S2, VBETA, D2 + vfmadd.d D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C0, 0x20 + vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_SUB X0, X0, K8 + PTR_ALSL X0, LDB, X0, 2 + move A0, A +.L_M8_END: + PTR_ADDI M8, M8, -1 + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 + bnez M8, .L_M8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M4_N3 +.L_M4_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M4_N4_END +.L_M4_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vldrepl.d Z0, X0, 0x00 + vldrepl.d Z1, B1, 0x00 + vldrepl.d Z2, B2, 0x00 + vldrepl.d Z3, B3, 0x00 + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S0, Z1, D2 + vfmadd.d D3, S1, Z1, D3 + vfmadd.d D4, S0, Z2, D4 + vfmadd.d D5, S1, Z2, D5 + vfmadd.d D6, S0, Z3, D6 + vfmadd.d D7, S1, Z3, D7 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADDI B2, B2, 0x08 + PTR_ADDI B3, B3, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M4_N4_K1 +.L_M4_N4_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA + vfmul.d D2, D2, VALPHA + vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA + vfmul.d D5, D5, VALPHA + vfmul.d D6, D6, VALPHA + vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 + vld S0, C2, 0x00; vld S1, C2, 0x10; vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5 + vld S0, C3, 0x00; vld S1, C3, 0x10; vfmadd.d D6, S0, VBETA, D6; vfmadd.d D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C1, 0x00 + vst D3, C1, 0x10 + vst D4, C2, 0x00 + vst D5, C2, 0x10 + vst D6, C3, 0x00 + vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_SUB B2, B2, K8 + PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M4_N4 +.L_M4_N3: + andi N2, N, 0x02 + beqz N2, .L_M4_N1 +.L_M4_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M4_N2_END +.L_M4_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vldrepl.d Z0, X0, 0x00 + vldrepl.d Z1, B1, 0x00 + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S0, Z1, D2 + vfmadd.d D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI B1, B1, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M4_N2_K1 +.L_M4_N2_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA + vfmul.d D2, D2, VALPHA + vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C1, 0x00 + vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K8 + PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M4_N1: + andi N1, N, 0x01 + beqz N1, .L_M4_END + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_M4_N1_END +.L_M4_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0 + vfmadd.d D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADD A0, A0, LDA + bnez K1, .L_M4_N1_K1 +.L_M4_N1_END: + vfmul.d D0, D0, VALPHA + vfmul.d D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 +.L_M4_END: + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + DGEMM_SMALL_KERNEL_NN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + DGEMM_SMALL_KERNEL_NN_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/dgemm_small_kernel_nt_lsx.S b/kernel/loongarch64/dgemm_small_kernel_nt_lsx.S new file mode 100644 index 0000000000..fdcde53d54 --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_nt_lsx.S @@ -0,0 +1,468 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define TEMP $t3 +#define A0 $t4 +#define X0 $t5 +#define B_cur $t6 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 + +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro DGEMM_SMALL_KERNEL_NT_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_NT_M\M\()_N3 +.L_NT_M\M\()_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_NT_M\M\()_N4_END +.L_NT_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vld S0, A0, 0x00 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08; vldrepl.d Z2, X0, 0x10; vldrepl.d Z3, X0, 0x18 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S0, Z1, D1; vfmadd.d D2, S0, Z2, D2; vfmadd.d D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_NT_M\M\()_N4_K1 +.L_NT_M\M\()_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2; vld S0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2; fld.d FS0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00; vst V1, C1, 0x00; vst V2, C2, 0x00; vst V3, C3, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00; fst.d F1, C1, 0x00; fst.d F2, C2, 0x00; fst.d F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 32 + move A0, A + bnez N4, .L_NT_M\M\()_N4 +.L_NT_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_NT_M\M\()_N1 +.L_NT_M\M\()_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_NT_M\M\()_N2_END +.L_NT_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vld S0, A0, 0x00 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_NT_M\M\()_N2_K1 +.L_NT_M\M\()_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00; vst V1, C1, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00; fst.d F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 16 + move A0, A +.L_NT_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_NT_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + move X0, B_cur + bge ZERO, K, .L_NT_M\M\()_N1_END +.L_NT_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vld S0, A0, 0x00 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_NT_M\M\()_N1_K1 +.L_NT_M\M\()_N1_END: + vfmul.d D0, D0, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00 +.endif +.L_NT_M\M\()_END: +.if \M == 2 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 1 + PTR_ADDI A, A, 0x08 + PTR_ADDI C, C, 0x08 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + vreplvei.d VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.d VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_SRAI M8, M, 3 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + vxor.v D8, D8, D8; vxor.v D9, D9, D9; vxor.v D10, D10, D10; vxor.v D11, D11, D11 + vxor.v D12, D12, D12; vxor.v D13, D13, D13; vxor.v D14, D14, D14; vxor.v D15, D15, D15 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10; vld S2, A0, 0x20; vld S3, A0, 0x30 + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08; vldrepl.d Z2, X0, 0x10; vldrepl.d Z3, X0, 0x18 + + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + vfmadd.d D4, S0, Z1, D4; vfmadd.d D5, S1, Z1, D5; vfmadd.d D6, S2, Z1, D6; vfmadd.d D7, S3, Z1, D7 + vfmadd.d D8, S0, Z2, D8; vfmadd.d D9, S1, Z2, D9; vfmadd.d D10, S2, Z2, D10; vfmadd.d D11, S3, Z2, D11 + vfmadd.d D12, S0, Z3, D12; vfmadd.d D13, S1, Z3, D13; vfmadd.d D14, S2, Z3, D14; vfmadd.d D15, S3, Z3, D15 + + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA + vfmul.d D8, D8, VALPHA; vfmul.d D9, D9, VALPHA; vfmul.d D10, D10, VALPHA; vfmul.d D11, D11, VALPHA + vfmul.d D12, D12, VALPHA; vfmul.d D13, D13, VALPHA; vfmul.d D14, D14, VALPHA; vfmul.d D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5; vfmadd.d D6, S2, VBETA, D6; vfmadd.d D7, S3, VBETA, D7 + vld S0, C2, 0x00; vld S1, C2, 0x10; vld S2, C2, 0x20; vld S3, C2, 0x30 + vfmadd.d D8, S0, VBETA, D8; vfmadd.d D9, S1, VBETA, D9; vfmadd.d D10, S2, VBETA, D10; vfmadd.d D11, S3, VBETA, D11 + vld S0, C3, 0x00; vld S1, C3, 0x10; vld S2, C3, 0x20; vld S3, C3, 0x30 + vfmadd.d D12, S0, VBETA, D12; vfmadd.d D13, S1, VBETA, D13; vfmadd.d D14, S2, VBETA, D14; vfmadd.d D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + vst D8, C2, 0x00; vst D9, C2, 0x10; vst D10, C2, 0x20; vst D11, C2, 0x30 + vst D12, C3, 0x00; vst D13, C3, 0x10; vst D14, C3, 0x20; vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 32 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10; vld S2, A0, 0x20; vld S3, A0, 0x30 + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + vfmadd.d D4, S0, Z1, D4; vfmadd.d D5, S1, Z1, D5; vfmadd.d D6, S2, Z1, D6; vfmadd.d D7, S3, Z1, D7 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5; vfmadd.d D6, S2, VBETA, D6; vfmadd.d D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 16 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10; vld S2, A0, 0x20; vld S3, A0, 0x30 + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M8_END: + PTR_ADDI M8, M8, -1 + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 + bnez M8, .L_M8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC; PTR_ADD C2, C1, LDC; PTR_ADD C3, C2, LDC + beqz N4, .L_M4_N3 +.L_M4_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M4_N4_END +.L_M4_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10 + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08; vldrepl.d Z2, X0, 0x10; vldrepl.d Z3, X0, 0x18 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S0, Z1, D2; vfmadd.d D3, S1, Z1, D3 + vfmadd.d D4, S0, Z2, D4; vfmadd.d D5, S1, Z2, D5 + vfmadd.d D6, S0, Z3, D6; vfmadd.d D7, S1, Z3, D7 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M4_N4_K1 +.L_M4_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 + vld S0, C2, 0x00; vld S1, C2, 0x10; vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5 + vld S0, C3, 0x00; vld S1, C3, 0x10; vfmadd.d D6, S0, VBETA, D6; vfmadd.d D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C1, 0x00; vst D3, C1, 0x10 + vst D4, C2, 0x00; vst D5, C2, 0x10; vst D6, C3, 0x00; vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2; PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 32 + move A0, A + bnez N4, .L_M4_N4 +.L_M4_N3: + andi N2, N, 0x02 + beqz N2, .L_M4_N1 +.L_M4_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M4_N2_END +.L_M4_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10 + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S0, Z1, D2; vfmadd.d D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M4_N2_K1 +.L_M4_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C1, 0x00; vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 16 + move A0, A +.L_M4_N1: + andi N1, N, 0x01 + beqz N1, .L_M4_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M4_N1_END +.L_M4_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10 + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M4_N1_K1 +.L_M4_N1_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 +.L_M4_END: + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + DGEMM_SMALL_KERNEL_NT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + DGEMM_SMALL_KERNEL_NT_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/dgemm_small_kernel_tn_lsx.S b/kernel/loongarch64/dgemm_small_kernel_tn_lsx.S new file mode 100644 index 0000000000..c68f07969b --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_tn_lsx.S @@ -0,0 +1,575 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define TEMP $t3 +#define K8 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro DGEMM_SMALL_KERNEL_TN_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_TN_M\M\()_N3 +.L_TN_M\M\()_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_TN_M\M\()_N4_END +.L_TN_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, B1, 0x00; vldrepl.d Z2, B2, 0x00; vldrepl.d Z3, B3, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S0, Z1, D1; vfmadd.d D2, S0, Z2, D2; vfmadd.d D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x08; PTR_ADDI B1, B1, 0x08; PTR_ADDI B2, B2, 0x08; PTR_ADDI B3, B3, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_TN_M\M\()_N4_K1 +.L_TN_M\M\()_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2; vld S0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2; fld.d FS0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00; vst V1, C1, 0x00; vst V2, C2, 0x00; vst V3, C3, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00; fst.d F1, C1, 0x00; fst.d F2, C2, 0x00; fst.d F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_SUB B1, B1, K8; PTR_SUB B2, B2, K8; PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2; PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2; PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_TN_M\M\()_N4 +.L_TN_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_TN_M\M\()_N1 +.L_TN_M\M\()_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_TN_M\M\()_N2_END +.L_TN_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, B1, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x08; PTR_ADDI B1, B1, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_TN_M\M\()_N2_K1 +.L_TN_M\M\()_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00; vst V1, C1, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00; fst.d F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1; PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_TN_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_TN_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + bge ZERO, K, .L_TN_M\M\()_N1_END +.L_TN_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_TN_M\M\()_N1_K1 +.L_TN_M\M\()_N1_END: + vfmul.d D0, D0, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00 +.endif +.L_TN_M\M\()_END: +.if \M == 2 + PTR_ALSL A, LDA, A, 1 + PTR_ADDI C, C, 0x10 +.elseif \M == 1 + PTR_ADD A, A, LDA + PTR_ADDI C, C, 0x08 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + vreplvei.d VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.d VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_SRAI M8, M, 3 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + vxor.v D8, D8, D8; vxor.v D9, D9, D9; vxor.v D10, D10, D10; vxor.v D11, D11, D11 + vxor.v D12, D12, D12; vxor.v D13, D13, D13; vxor.v D14, D14, D14; vxor.v D15, D15, D15 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S2, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S2, Z0, S2 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S3, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S3, Z0, S3 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, B1, 0x00; vldrepl.d Z2, B2, 0x00; vldrepl.d Z3, B3, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + vfmadd.d D4, S0, Z1, D4; vfmadd.d D5, S1, Z1, D5; vfmadd.d D6, S2, Z1, D6; vfmadd.d D7, S3, Z1, D7 + vfmadd.d D8, S0, Z2, D8; vfmadd.d D9, S1, Z2, D9; vfmadd.d D10, S2, Z2, D10; vfmadd.d D11, S3, Z2, D11 + vfmadd.d D12, S0, Z3, D12; vfmadd.d D13, S1, Z3, D13; vfmadd.d D14, S2, Z3, D14; vfmadd.d D15, S3, Z3, D15 + + PTR_ADDI X0, X0, 0x08; PTR_ADDI B1, B1, 0x08; PTR_ADDI B2, B2, 0x08; PTR_ADDI B3, B3, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA + vfmul.d D8, D8, VALPHA; vfmul.d D9, D9, VALPHA; vfmul.d D10, D10, VALPHA; vfmul.d D11, D11, VALPHA + vfmul.d D12, D12, VALPHA; vfmul.d D13, D13, VALPHA; vfmul.d D14, D14, VALPHA; vfmul.d D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5; vfmadd.d D6, S2, VBETA, D6; vfmadd.d D7, S3, VBETA, D7 + vld S0, C2, 0x00; vld S1, C2, 0x10; vld S2, C2, 0x20; vld S3, C2, 0x30 + vfmadd.d D8, S0, VBETA, D8; vfmadd.d D9, S1, VBETA, D9; vfmadd.d D10, S2, VBETA, D10; vfmadd.d D11, S3, VBETA, D11 + vld S0, C3, 0x00; vld S1, C3, 0x10; vld S2, C3, 0x20; vld S3, C3, 0x30 + vfmadd.d D12, S0, VBETA, D12; vfmadd.d D13, S1, VBETA, D13; vfmadd.d D14, S2, VBETA, D14; vfmadd.d D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + vst D8, C2, 0x00; vst D9, C2, 0x10; vst D10, C2, 0x20; vst D11, C2, 0x30 + vst D12, C3, 0x00; vst D13, C3, 0x10; vst D14, C3, 0x20; vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_SUB B1, B1, K8; PTR_SUB B2, B2, K8; PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2; PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2; PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S2, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S2, Z0, S2 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S3, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S3, Z0, S3 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, B1, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + vfmadd.d D4, S0, Z1, D4; vfmadd.d D5, S1, Z1, D5; vfmadd.d D6, S2, Z1, D6; vfmadd.d D7, S3, Z1, D7 + + PTR_ADDI X0, X0, 0x08; PTR_ADDI B1, B1, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5; vfmadd.d D6, S2, VBETA, D6; vfmadd.d D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1; PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S2, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S2, Z0, S2 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S3, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S3, Z0, S3 + + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_ALSL X0, LDB, X0, 2 + move A0, A +.L_M8_END: + PTR_ADDI M8, M8, -1 + PTR_ALSL A, LDA, A, 3 + PTR_ADDI C, C, 0x40 + bnez M8, .L_M8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB; PTR_ADD B2, B1, LDB; PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC; PTR_ADD C2, C1, LDC; PTR_ADD C3, C2, LDC + beqz N4, .L_M4_N3 +.L_M4_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M4_N4_END +.L_M4_N4_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, B1, 0x00; vldrepl.d Z2, B2, 0x00; vldrepl.d Z3, B3, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S0, Z1, D2; vfmadd.d D3, S1, Z1, D3 + vfmadd.d D4, S0, Z2, D4; vfmadd.d D5, S1, Z2, D5; vfmadd.d D6, S0, Z3, D6; vfmadd.d D7, S1, Z3, D7 + PTR_ADDI X0, X0, 0x08; PTR_ADDI B1, B1, 0x08; PTR_ADDI B2, B2, 0x08; PTR_ADDI B3, B3, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M4_N4_K1 +.L_M4_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 + vld S0, C2, 0x00; vld S1, C2, 0x10; vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5 + vld S0, C3, 0x00; vld S1, C3, 0x10; vfmadd.d D6, S0, VBETA, D6; vfmadd.d D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C1, 0x00; vst D3, C1, 0x10 + vst D4, C2, 0x00; vst D5, C2, 0x10; vst D6, C3, 0x00; vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2; PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_SUB B1, B1, K8; PTR_SUB B2, B2, K8; PTR_SUB B3, B3, K8 + PTR_ALSL X0, LDB, X0, 2; PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2; PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M4_N4 +.L_M4_N3: + andi N2, N, 0x02 + beqz N2, .L_M4_N1 +.L_M4_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M4_N2_END +.L_M4_N2_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, B1, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S0, Z1, D2; vfmadd.d D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x08; PTR_ADDI B1, B1, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M4_N2_K1 +.L_M4_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C1, 0x00; vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_SLLI K8, K, 3 + PTR_SUB X0, X0, K8; PTR_SUB B1, B1, K8 + PTR_ALSL X0, LDB, X0, 1; PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M4_N1: + andi N1, N, 0x01 + beqz N1, .L_M4_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_M4_N1_END +.L_M4_N1_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x08 + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M4_N1_K1 +.L_M4_N1_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 +.L_M4_END: + PTR_ALSL A, LDA, A, 2 + PTR_ADDI C, C, 0x20 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + DGEMM_SMALL_KERNEL_TN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + DGEMM_SMALL_KERNEL_TN_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/dgemm_small_kernel_tt_lsx.S b/kernel/loongarch64/dgemm_small_kernel_tt_lsx.S new file mode 100644 index 0000000000..b5484aaa6f --- /dev/null +++ b/kernel/loongarch64/dgemm_small_kernel_tt_lsx.S @@ -0,0 +1,559 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define TEMP $t3 +#define A0 $t4 +#define X0 $t5 +#define B_cur $t6 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro DGEMM_SMALL_KERNEL_TT_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_TT_M\M\()_N3 +.L_TT_M\M\()_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_TT_M\M\()_N4_END +.L_TT_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08; vldrepl.d Z2, X0, 0x10; vldrepl.d Z3, X0, 0x18 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S0, Z1, D1; vfmadd.d D2, S0, Z2, D2; vfmadd.d D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_TT_M\M\()_N4_K1 +.L_TT_M\M\()_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2; vld S0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.d D2, S0, VBETA, D2; fld.d FS0, C3, 0x00; vfmadd.d D3, S0, VBETA, D3 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00; vst V1, C1, 0x00; vst V2, C2, 0x00; vst V3, C3, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00; fst.d F1, C1, 0x00; fst.d F2, C2, 0x00; fst.d F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 32 + move A0, A + bnez N4, .L_TT_M\M\()_N4 +.L_TT_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_TT_M\M\()_N1 +.L_TT_M\M\()_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_TT_M\M\()_N2_END +.L_TT_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_TT_M\M\()_N2_K1 +.L_TT_M\M\()_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; vld S0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0; fld.d FS0, C1, 0x00; vfmadd.d D1, S0, VBETA, D1 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00; vst V1, C1, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00; fst.d F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 16 + move A0, A +.L_TT_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_TT_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + move X0, B_cur + bge ZERO, K, .L_TT_M\M\()_N1_END +.L_TT_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 +.if \M == 2 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 +.elseif \M == 1 + fld.d FS0, A0, 0x00 +.endif + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_TT_M\M\()_N1_K1 +.L_TT_M\M\()_N1_END: + vfmul.d D0, D0, VALPHA +#ifndef B0 +.if \M == 2 + vld S0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.elseif \M == 1 + fld.d FS0, C0, 0x00; vfmadd.d D0, S0, VBETA, D0 +.endif +#endif +.if \M == 2 + vst V0, C0, 0x00 +.elseif \M == 1 + fst.d F0, C0, 0x00 +.endif +.L_TT_M\M\()_END: +.if \M == 2 + PTR_ALSL A, LDA, A, 1 + PTR_ADDI C, C, 0x10 +.elseif \M == 1 + PTR_ADD A, A, LDA + PTR_ADDI C, C, 0x08 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + vreplvei.d VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.d VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 3 + PTR_SLLI LDB, LDB, 3 + PTR_SLLI LDC, LDC, 3 + PTR_SRAI M8, M, 3 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + vxor.v D8, D8, D8; vxor.v D9, D9, D9; vxor.v D10, D10, D10; vxor.v D11, D11, D11 + vxor.v D12, D12, D12; vxor.v D13, D13, D13; vxor.v D14, D14, D14; vxor.v D15, D15, D15 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S2, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S2, Z0, S2 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S3, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S3, Z0, S3 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08; vldrepl.d Z2, X0, 0x10; vldrepl.d Z3, X0, 0x18 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + vfmadd.d D4, S0, Z1, D4; vfmadd.d D5, S1, Z1, D5; vfmadd.d D6, S2, Z1, D6; vfmadd.d D7, S3, Z1, D7 + vfmadd.d D8, S0, Z2, D8; vfmadd.d D9, S1, Z2, D9; vfmadd.d D10, S2, Z2, D10; vfmadd.d D11, S3, Z2, D11 + vfmadd.d D12, S0, Z3, D12; vfmadd.d D13, S1, Z3, D13; vfmadd.d D14, S2, Z3, D14; vfmadd.d D15, S3, Z3, D15 + + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA + vfmul.d D8, D8, VALPHA; vfmul.d D9, D9, VALPHA; vfmul.d D10, D10, VALPHA; vfmul.d D11, D11, VALPHA + vfmul.d D12, D12, VALPHA; vfmul.d D13, D13, VALPHA; vfmul.d D14, D14, VALPHA; vfmul.d D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5; vfmadd.d D6, S2, VBETA, D6; vfmadd.d D7, S3, VBETA, D7 + vld S0, C2, 0x00; vld S1, C2, 0x10; vld S2, C2, 0x20; vld S3, C2, 0x30 + vfmadd.d D8, S0, VBETA, D8; vfmadd.d D9, S1, VBETA, D9; vfmadd.d D10, S2, VBETA, D10; vfmadd.d D11, S3, VBETA, D11 + vld S0, C3, 0x00; vld S1, C3, 0x10; vld S2, C3, 0x20; vld S3, C3, 0x30 + vfmadd.d D12, S0, VBETA, D12; vfmadd.d D13, S1, VBETA, D13; vfmadd.d D14, S2, VBETA, D14; vfmadd.d D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + vst D8, C2, 0x00; vst D9, C2, 0x10; vst D10, C2, 0x20; vst D11, C2, 0x30 + vst D12, C3, 0x00; vst D13, C3, 0x10; vst D14, C3, 0x20; vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 32 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S2, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S2, Z0, S2 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S3, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S3, Z0, S3 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + vfmadd.d D4, S0, Z1, D4; vfmadd.d D5, S1, Z1, D5; vfmadd.d D6, S2, Z1, D6; vfmadd.d D7, S3, Z1, D7 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5; vfmadd.d D6, S2, VBETA, D6; vfmadd.d D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 16 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S2, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S2, Z0, S2 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S3, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S3, Z0, S3 + + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1; vfmadd.d D2, S2, Z0, D2; vfmadd.d D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1; vfmadd.d D2, S2, VBETA, D2; vfmadd.d D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M8_END: + PTR_ADDI M8, M8, -1 + PTR_ALSL A, LDA, A, 3 + PTR_ADDI C, C, 0x40 + bnez M8, .L_M8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC; PTR_ADD C2, C1, LDC; PTR_ADD C3, C2, LDC + beqz N4, .L_M4_N3 +.L_M4_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M4_N4_END +.L_M4_N4_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08; vldrepl.d Z2, X0, 0x10; vldrepl.d Z3, X0, 0x18 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S0, Z1, D2; vfmadd.d D3, S1, Z1, D3 + vfmadd.d D4, S0, Z2, D4; vfmadd.d D5, S1, Z2, D5 + vfmadd.d D6, S0, Z3, D6; vfmadd.d D7, S1, Z3, D7 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M4_N4_K1 +.L_M4_N4_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA + vfmul.d D4, D4, VALPHA; vfmul.d D5, D5, VALPHA; vfmul.d D6, D6, VALPHA; vfmul.d D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 + vld S0, C2, 0x00; vld S1, C2, 0x10; vfmadd.d D4, S0, VBETA, D4; vfmadd.d D5, S1, VBETA, D5 + vld S0, C3, 0x00; vld S1, C3, 0x10; vfmadd.d D6, S0, VBETA, D6; vfmadd.d D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C1, 0x00; vst D3, C1, 0x10 + vst D4, C2, 0x00; vst D5, C2, 0x10; vst D6, C3, 0x00; vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2; PTR_ALSL C1, LDC, C1, 2; PTR_ALSL C2, LDC, C2, 2; PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 32 + move A0, A + bnez N4, .L_M4_N4 +.L_M4_N3: + andi N2, N, 0x02 + beqz N2, .L_M4_N1 +.L_M4_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M4_N2_END +.L_M4_N2_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + + vldrepl.d Z0, X0, 0x00; vldrepl.d Z1, X0, 0x08 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + vfmadd.d D2, S0, Z1, D2; vfmadd.d D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M4_N2_K1 +.L_M4_N2_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA; vfmul.d D2, D2, VALPHA; vfmul.d D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 + vld S0, C1, 0x00; vld S1, C1, 0x10; vfmadd.d D2, S0, VBETA, D2; vfmadd.d D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C1, 0x00; vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1; PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 16 + move A0, A +.L_M4_N1: + andi N1, N, 0x01 + beqz N1, .L_M4_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M4_N1_END +.L_M4_N1_K1: + PTR_ADDI K1, K1, -1 + vldrepl.d S0, A0, 0 + PTR_ADD TEMP, A0, LDA + vldrepl.d S1, TEMP, 0 + vilvl.d S0, S1, S0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d S1, TEMP, 0 + PTR_ADD TEMP, TEMP, LDA + vldrepl.d Z0, TEMP, 0 + vilvl.d S1, Z0, S1 + + vldrepl.d Z0, X0, 0x00 + vfmadd.d D0, S0, Z0, D0; vfmadd.d D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x08 + bnez K1, .L_M4_N1_K1 +.L_M4_N1_END: + vfmul.d D0, D0, VALPHA; vfmul.d D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vfmadd.d D0, S0, VBETA, D0; vfmadd.d D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 +.L_M4_END: + PTR_ALSL A, LDA, A, 2 + PTR_ADDI C, C, 0x20 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + DGEMM_SMALL_KERNEL_TT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + DGEMM_SMALL_KERNEL_TT_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/dgemm_small_matrix_permit.c b/kernel/loongarch64/dgemm_small_matrix_permit.c index df262a6bb3..b4c85ac53c 100644 --- a/kernel/loongarch64/dgemm_small_matrix_permit.c +++ b/kernel/loongarch64/dgemm_small_matrix_permit.c @@ -31,6 +31,7 @@ int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alph { double MNK = (double) M * (double) N * (double) K; +#ifdef LA464 if (transa) { if (MNK <= 24.0 * 24.0 * 24.0) return 1; @@ -38,6 +39,17 @@ int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alph if (MNK <= 64.0 * 64.0 * 64.0) return 1; } +#endif + +#ifdef LA264 + if (transa) { + if (MNK <= 24.0 * 24.0 * 24.0) + return 1; + } else { + if (MNK <= 80.0 * 80.0 * 80.0) + return 1; + } +#endif return 0; } diff --git a/kernel/loongarch64/sgemm_small_kernel_nn_lasx.S b/kernel/loongarch64/sgemm_small_kernel_nn_lasx.S new file mode 100644 index 0000000000..4ae23debbb --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_nn_lasx.S @@ -0,0 +1,566 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M32 $t1 +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define D4 $xr6 +#define D5 $xr7 +#define D6 $xr8 +#define D7 $xr9 +#define D8 $xr10 +#define D9 $xr11 +#define D10 $xr12 +#define D11 $xr13 +#define D12 $xr14 +#define D13 $xr15 +#define D14 $xr16 +#define D15 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + +.macro SGEMM_SMALL_KERNEL_NN_TAIL M + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M\M\()_N3 +.L_M\M\()_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M\M\()_N4_END +.L_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N4_K1 +.L_M\M\()_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 + GLD xv, , S0, C2, 0x00 + GMADD xvf, s, D2, S0, VBETA, D2 + GLD xv, , S0, C3, 0x00 + GMADD xvf, s, D3, S0, VBETA, D3 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.endif + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0 + move A0, A + bnez N4, .L_M\M\()_N4 +.L_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_M\M\()_N1 +.L_M\M\()_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M\M\()_N2_END +.L_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N2_K1 +.L_M\M\()_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00 +.endif + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_M\M\()_END + GXOR xv, v, D0, D0, D0 + move K1, K // Restore K1 + bge ZERO, K, .L_M\M\()_N1_END +.L_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N1_K1 +.L_M\M\()_N1_END: + GMUL xvf, s, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00 +.endif +.L_M\M\()_END: +.if \M == 8 + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.elseif \M == 4 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_ADDI A, A, 0x08 + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADDI A, A, 0x04 + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + xvreplve0.w VALPHA, VALPHA +#ifndef B0 + xvreplve0.w VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SLLI K4, K, 2 + PTR_SRAI M32, M, 5 // M >> 5 + beqz M32, .L_M31 +.L_M32: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M32_N3 +.L_M32_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7, \ + D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \ + D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M32_N4_END +.L_M32_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7, \ + D8, S0, Z2, D8, D9, S1, Z2, D9, D10, S2, Z2, D10, D11, S3, Z2, D11, \ + D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M32_N4_K1 + .L_M32_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA, \ + D8, D8, VALPHA, D9, D9, VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \ + D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60 + GMADD xvf, s, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60 + GMADD xvf, s, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15 +#endif + GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \ + D8, C2, 0x00, D9, C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \ + D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0 + move A0, A + bnez N4, .L_M32_N4 +.L_M32_N3: + andi N2, N, 0x02 + beqz N2, .L_M32_N1 +.L_M32_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K // Restore K1 + bge ZERO, K, .L_M32_N2_END +.L_M32_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M32_N2_K1 +.L_M32_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 +#endif + GST xv, , D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M32_N1: + andi N1, N, 0x01 + beqz N1, .L_M32_END + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + bge ZERO, K, .L_M32_N1_END +.L_M32_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M32_N1_K1 +.L_M32_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K4 + PTR_ALSL X0, LDB, X0, 2 + // Restore A0 + move A0, A +.L_M32_END: + PTR_ADDI M32, M32, -1 + PTR_ADDI A, A, 0x80 + PTR_ADDI C, C, 0x80 + bnez M32, .L_M32 +.L_M31: + andi M16, M, 0x10 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A // Restore A0 + move X0, B // Restore X0 + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C // Restore C0 + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K // Restore K1 + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3, \ + D4, S0, Z2, D4, D5, S1, Z2, D5, \ + D6, S0, Z3, D6, D7, S1, Z3, D7, + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20 + GMADD xvf, s, D6, S0, VBETA, D6, D7, S1, VBETA, D7 +#endif + GST xv, , D4, C2, 0x00, D5, C2, 0x20, D6, C3, 0x00, D7, C3, 0x20, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + // Update C0, C1, C2, C3 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update X0, B1, B2, B3 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + // Restore A0 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K // Restore K1 + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + // Update C0, C1 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + // Update X0, B1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + // Restore A0 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K // Restore K1 + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20 +.L_M16_END: + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + SGEMM_SMALL_KERNEL_NN_TAIL 8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_NN_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_NN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_NN_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_nn_lsx.S b/kernel/loongarch64/sgemm_small_kernel_nn_lsx.S new file mode 100644 index 0000000000..b480c424a5 --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_nn_lsx.S @@ -0,0 +1,795 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 + +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro SGEMM_SMALL_KERNEL_NN_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M\M\()_N3 +.L_M\M\()_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M\M\()_N4_END +.L_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vldrepl.w Z2, B2, 0x00 + vldrepl.w Z3, B3, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + vfmadd.s D2, S0, Z2, D2 + vfmadd.s D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N4_K1 +.L_M\M\()_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + vld S0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.d FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.s FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.s FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 + vst V2, C2, 0x00 + vst V3, C3, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 + fst.d F2, C2, 0x00 + fst.d F3, C3, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 + fst.s F2, C2, 0x00 + fst.s F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M\M\()_N4 +.L_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_M\M\()_N1 +.L_M\M\()_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_M\M\()_N2_END +.L_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N2_K1 +.L_M\M\()_N2_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + bge ZERO, K, .L_M\M\()_N1_END +.L_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N1_K1 +.L_M\M\()_N1_END: + vfmul.s D0, D0, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 +.endif +.L_M\M\()_END: +.if \M == 4 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_ADDI A, A, 0x08 + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADDI A, A, 0x04 + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + + vreplvei.w VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.w VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SLLI K4, K, 2 + PTR_SRAI M16, M, 4 // M >> 4 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 // N >> 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + vxor.v D8, D8, D8 + vxor.v D9, D9, D9 + vxor.v D10, D10, D10 + vxor.v D11, D11, D11 + vxor.v D12, D12, D12 + vxor.v D13, D13, D13 + vxor.v D14, D14, D14 + vxor.v D15, D15, D15 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vldrepl.w Z2, B2, 0x00 + vldrepl.w Z3, B3, 0x00 + + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S2, Z0, D2 + vfmadd.s D3, S3, Z0, D3 + + vfmadd.s D4, S0, Z1, D4 + vfmadd.s D5, S1, Z1, D5 + vfmadd.s D6, S2, Z1, D6 + vfmadd.s D7, S3, Z1, D7 + + vfmadd.s D8, S0, Z2, D8 + vfmadd.s D9, S1, Z2, D9 + vfmadd.s D10, S2, Z2, D10 + vfmadd.s D11, S3, Z2, D11 + + vfmadd.s D12, S0, Z3, D12 + vfmadd.s D13, S1, Z3, D13 + vfmadd.s D14, S2, Z3, D14 + vfmadd.s D15, S3, Z3, D15 + + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA + vfmul.s D5, D5, VALPHA + vfmul.s D6, D6, VALPHA + vfmul.s D7, D7, VALPHA + vfmul.s D8, D8, VALPHA + vfmul.s D9, D9, VALPHA + vfmul.s D10, D10, VALPHA + vfmul.s D11, D11, VALPHA + vfmul.s D12, D12, VALPHA + vfmul.s D13, D13, VALPHA + vfmul.s D14, D14, VALPHA + vfmul.s D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vld S2, C0, 0x20 + vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0 + vfmadd.s D1, S1, VBETA, D1 + vfmadd.s D2, S2, VBETA, D2 + vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00 + vld S1, C1, 0x10 + vld S2, C1, 0x20 + vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4 + vfmadd.s D5, S1, VBETA, D5 + vfmadd.s D6, S2, VBETA, D6 + vfmadd.s D7, S3, VBETA, D7 + + vld S0, C2, 0x00 + vld S1, C2, 0x10 + vld S2, C2, 0x20 + vld S3, C2, 0x30 + vfmadd.s D8, S0, VBETA, D8 + vfmadd.s D9, S1, VBETA, D9 + vfmadd.s D10, S2, VBETA, D10 + vfmadd.s D11, S3, VBETA, D11 + + vld S0, C3, 0x00 + vld S1, C3, 0x10 + vld S2, C3, 0x20 + vld S3, C3, 0x30 + vfmadd.s D12, S0, VBETA, D12 + vfmadd.s D13, S1, VBETA, D13 + vfmadd.s D14, S2, VBETA, D14 + vfmadd.s D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C0, 0x20 + vst D3, C0, 0x30 + vst D4, C1, 0x00 + vst D5, C1, 0x10 + vst D6, C1, 0x20 + vst D7, C1, 0x30 + vst D8, C2, 0x00 + vst D9, C2, 0x10 + vst D10, C2, 0x20 + vst D11, C2, 0x30 + vst D12, C3, 0x00 + vst D13, C3, 0x10 + vst D14, C3, 0x20 + vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + move K1, K + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S2, Z0, D2 + vfmadd.s D3, S3, Z0, D3 + + vfmadd.s D4, S0, Z1, D4 + vfmadd.s D5, S1, Z1, D5 + vfmadd.s D6, S2, Z1, D6 + vfmadd.s D7, S3, Z1, D7 + + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA + vfmul.s D5, D5, VALPHA + vfmul.s D6, D6, VALPHA + vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vld S2, C0, 0x20 + vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0 + vfmadd.s D1, S1, VBETA, D1 + vfmadd.s D2, S2, VBETA, D2 + vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00 + vld S1, C1, 0x10 + vld S2, C1, 0x20 + vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4 + vfmadd.s D5, S1, VBETA, D5 + vfmadd.s D6, S2, VBETA, D6 + vfmadd.s D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C0, 0x20 + vst D3, C0, 0x30 + vst D4, C1, 0x00 + vst D5, C1, 0x10 + vst D6, C1, 0x20 + vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S2, Z0, D2 + vfmadd.s D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vld S2, C0, 0x20 + vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0 + vfmadd.s D1, S1, VBETA, D1 + vfmadd.s D2, S2, VBETA, D2 + vfmadd.s D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C0, 0x20 + vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_SUB X0, X0, K4 + PTR_ALSL X0, LDB, X0, 2 + move A0, A +.L_M16_END: + PTR_ADDI M16, M16, -1 + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 + bnez M16, .L_M16 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vldrepl.w Z2, B2, 0x00 + vldrepl.w Z3, B3, 0x00 + + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2 + vfmadd.s D3, S1, Z1, D3 + vfmadd.s D4, S0, Z2, D4 + vfmadd.s D5, S1, Z2, D5 + vfmadd.s D6, S0, Z3, D6 + vfmadd.s D7, S1, Z3, D7 + + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA + vfmul.s D5, D5, VALPHA + vfmul.s D6, D6, VALPHA + vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0 + vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00 + vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2 + vfmadd.s D3, S1, VBETA, D3 + + vld S0, C2, 0x00 + vld S1, C2, 0x10 + vfmadd.s D4, S0, VBETA, D4 + vfmadd.s D5, S1, VBETA, D5 + + vld S0, C3, 0x00 + vld S1, C3, 0x10 + vfmadd.s D6, S0, VBETA, D6 + vfmadd.s D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C1, 0x00 + vst D3, C1, 0x10 + vst D4, C2, 0x00 + vst D5, C2, 0x10 + vst D6, C3, 0x00 + vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2 + vfmadd.s D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0 + vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00 + vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2 + vfmadd.s D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 + vst D2, C1, 0x00 + vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00 + vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0 + vfmadd.s D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00 + vst D1, C0, 0x10 +.L_M8_END: + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_NN_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_NN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_NN_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_nt_lasx.S b/kernel/loongarch64/sgemm_small_kernel_nt_lasx.S new file mode 100644 index 0000000000..49827093aa --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_nt_lasx.S @@ -0,0 +1,499 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M32 $t1 +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B_cur $t6 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define D4 $xr6 +#define D5 $xr7 +#define D6 $xr8 +#define D7 $xr9 +#define D8 $xr10 +#define D9 $xr11 +#define D10 $xr12 +#define D11 $xr13 +#define D12 $xr14 +#define D13 $xr15 +#define D14 $xr16 +#define D15 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + +.macro SGEMM_SMALL_KERNEL_NT_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M\M\()_N3 +.L_M\M\()_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M\M\()_N4_END +.L_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04, Z2, X0, 0x08, Z3, X0, 0x0C + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N4_K1 +.L_M\M\()_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 + GLD xv, , S0, C2, 0x00 + GMADD xvf, s, D2, S0, VBETA, D2 + GLD xv, , S0, C3, 0x00 + GMADD xvf, s, D3, S0, VBETA, D3 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + // Update B_cur to advance N by 4 (16 bytes) + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M\M\()_N4 +.L_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_M\M\()_N1 +.L_M\M\()_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M\M\()_N2_END +.L_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04 + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N2_K1 +.L_M\M\()_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_M\M\()_END + GXOR xv, v, D0, D0, D0 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M\M\()_N1_END +.L_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00 + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M\M\()_N1_K1 +.L_M\M\()_N1_END: + GMUL xvf, s, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00 +.endif +.L_M\M\()_END: +.if \M == 8 + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.elseif \M == 4 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_ADDI A, A, 0x08 + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADDI A, A, 0x04 + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + xvreplve0.w VALPHA, VALPHA +#ifndef B0 + xvreplve0.w VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SRAI M32, M, 5 + beqz M32, .L_M31 +.L_M32: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B // Reset B to origin for each M block + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M32_N3 +.L_M32_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7, \ + D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \ + D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M32_N4_END +.L_M32_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04, Z2, X0, 0x08, Z3, X0, 0x0C + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7, \ + D8, S0, Z2, D8, D9, S1, Z2, D9, D10, S2, Z2, D10, D11, S3, Z2, D11, \ + D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M32_N4_K1 + .L_M32_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA, \ + D8, D8, VALPHA, D9, D9, VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \ + D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60 + GMADD xvf, s, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60 + GMADD xvf, s, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15 +#endif + GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \ + D8, C2, 0x00, D9, C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \ + D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 // Advance B along N by 4 words + move A0, A + bnez N4, .L_M32_N4 +.L_M32_N3: + andi N2, N, 0x02 + beqz N2, .L_M32_N1 +.L_M32_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M32_N2_END +.L_M32_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M32_N2_K1 +.L_M32_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 +#endif + GST xv, , D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M32_N1: + andi N1, N, 0x01 + beqz N1, .L_M32_END + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M32_N1_END +.L_M32_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60 + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M32_N1_K1 +.L_M32_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 2 + PTR_ADDI B_cur, B_cur, 4 + move A0, A +.L_M32_END: + PTR_ADDI M32, M32, -1 + PTR_ADDI A, A, 0x80 + PTR_ADDI C, C, 0x80 + bnez M32, .L_M32 +.L_M31: + andi M16, M, 0x10 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04, Z2, X0, 0x08, Z3, X0, 0x0C + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3, \ + D4, S0, Z2, D4, D5, S1, Z2, D5, \ + D6, S0, Z3, D6, D7, S1, Z3, D7 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20 + GMADD xvf, s, D6, S0, VBETA, D6, D7, S1, VBETA, D7 +#endif + GST xv, , D4, C2, 0x00, D5, C2, 0x20, D6, C3, 0x00, D7, C3, 0x20, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + GLD xv, , S0, A0, 0x00, S1, A0, 0x20 + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20 +.L_M16_END: + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + SGEMM_SMALL_KERNEL_NT_TAIL 8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_NT_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_NT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_NT_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_nt_lsx.S b/kernel/loongarch64/sgemm_small_kernel_nt_lsx.S new file mode 100644 index 0000000000..58506e5aeb --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_nt_lsx.S @@ -0,0 +1,614 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B_cur $t6 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 + +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro SGEMM_SMALL_KERNEL_NT_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_NT_M\M\()_N3 +.L_NT_M\M\()_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_NT_M\M\()_N4_END +.L_NT_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 +.if \M == 4 + vld S0, A0, 0x00 +.elseif \M == 2 + fld.d FS0, A0, 0x00 +.elseif \M == 1 + fld.s FS0, A0, 0x00 +.endif + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, X0, 0x04 + vldrepl.w Z2, X0, 0x08 + vldrepl.w Z3, X0, 0x0C + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + vfmadd.s D2, S0, Z2, D2 + vfmadd.s D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_NT_M\M\()_N4_K1 +.L_NT_M\M\()_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + vld S0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.d FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.s FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.s FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 + vst V2, C2, 0x00 + vst V3, C3, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 + fst.d F2, C2, 0x00 + fst.d F3, C3, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 + fst.s F2, C2, 0x00 + fst.s F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_NT_M\M\()_N4 +.L_NT_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_NT_M\M\()_N1 +.L_NT_M\M\()_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_NT_M\M\()_N2_END +.L_NT_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 +.if \M == 4 + vld S0, A0, 0x00 +.elseif \M == 2 + fld.d FS0, A0, 0x00 +.elseif \M == 1 + fld.s FS0, A0, 0x00 +.endif + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, X0, 0x04 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_NT_M\M\()_N2_K1 +.L_NT_M\M\()_N2_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_NT_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_NT_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + move X0, B_cur + bge ZERO, K, .L_NT_M\M\()_N1_END +.L_NT_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 +.if \M == 4 + vld S0, A0, 0x00 +.elseif \M == 2 + fld.d FS0, A0, 0x00 +.elseif \M == 1 + fld.s FS0, A0, 0x00 +.endif + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_NT_M\M\()_N1_K1 +.L_NT_M\M\()_N1_END: + vfmul.s D0, D0, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 +.endif +.L_NT_M\M\()_END: +.if \M == 4 + PTR_ADDI A, A, 0x10 + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_ADDI A, A, 0x08 + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADDI A, A, 0x04 + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 5, 2 + vreplvei.w VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.w VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SRAI M16, M, 4 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + vxor.v D4, D4, D4 + vxor.v D5, D5, D5 + vxor.v D6, D6, D6 + vxor.v D7, D7, D7 + vxor.v D8, D8, D8 + vxor.v D9, D9, D9 + vxor.v D10, D10, D10 + vxor.v D11, D11, D11 + vxor.v D12, D12, D12 + vxor.v D13, D13, D13 + vxor.v D14, D14, D14 + vxor.v D15, D15, D15 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00 + vld S1, A0, 0x10 + vld S2, A0, 0x20 + vld S3, A0, 0x30 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, X0, 0x04 + vldrepl.w Z2, X0, 0x08 + vldrepl.w Z3, X0, 0x0C + + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S2, Z0, D2 + vfmadd.s D3, S3, Z0, D3 + + vfmadd.s D4, S0, Z1, D4 + vfmadd.s D5, S1, Z1, D5 + vfmadd.s D6, S2, Z1, D6 + vfmadd.s D7, S3, Z1, D7 + + vfmadd.s D8, S0, Z2, D8 + vfmadd.s D9, S1, Z2, D9 + vfmadd.s D10, S2, Z2, D10 + vfmadd.s D11, S3, Z2, D11 + + vfmadd.s D12, S0, Z3, D12 + vfmadd.s D13, S1, Z3, D13 + vfmadd.s D14, S2, Z3, D14 + vfmadd.s D15, S3, Z3, D15 + + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA + vfmul.s D5, D5, VALPHA + vfmul.s D6, D6, VALPHA + vfmul.s D7, D7, VALPHA + vfmul.s D8, D8, VALPHA + vfmul.s D9, D9, VALPHA + vfmul.s D10, D10, VALPHA + vfmul.s D11, D11, VALPHA + vfmul.s D12, D12, VALPHA + vfmul.s D13, D13, VALPHA + vfmul.s D14, D14, VALPHA + vfmul.s D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5; vfmadd.s D6, S2, VBETA, D6; vfmadd.s D7, S3, VBETA, D7 + + vld S0, C2, 0x00; vld S1, C2, 0x10; vld S2, C2, 0x20; vld S3, C2, 0x30 + vfmadd.s D8, S0, VBETA, D8; vfmadd.s D9, S1, VBETA, D9; vfmadd.s D10, S2, VBETA, D10; vfmadd.s D11, S3, VBETA, D11 + + vld S0, C3, 0x00; vld S1, C3, 0x10; vld S2, C3, 0x20; vld S3, C3, 0x30 + vfmadd.s D12, S0, VBETA, D12; vfmadd.s D13, S1, VBETA, D13; vfmadd.s D14, S2, VBETA, D14; vfmadd.s D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + vst D8, C2, 0x00; vst D9, C2, 0x10; vst D10, C2, 0x20; vst D11, C2, 0x30 + vst D12, C3, 0x00; vst D13, C3, 0x10; vst D14, C3, 0x20; vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10; vld S2, A0, 0x20; vld S3, A0, 0x30 + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, X0, 0x04 + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + vfmadd.s D4, S0, Z1, D4; vfmadd.s D5, S1, Z1, D5; vfmadd.s D6, S2, Z1, D6; vfmadd.s D7, S3, Z1, D7 + + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5; vfmadd.s D6, S2, VBETA, D6; vfmadd.s D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10; vld S2, A0, 0x20; vld S3, A0, 0x30 + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ADDI B_cur, B_cur, 4 + move A0, A +.L_M16_END: + PTR_ADDI M16, M16, -1 + PTR_ADDI A, A, 0x40 + PTR_ADDI C, C, 0x40 + bnez M16, .L_M16 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10 + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, X0, 0x04; vldrepl.w Z2, X0, 0x08; vldrepl.w Z3, X0, 0x0C + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2; vfmadd.s D3, S1, Z1, D3 + vfmadd.s D4, S0, Z2, D4; vfmadd.s D5, S1, Z2, D5 + vfmadd.s D6, S0, Z3, D6; vfmadd.s D7, S1, Z3, D7 + + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00; vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2; vfmadd.s D3, S1, VBETA, D3 + + vld S0, C2, 0x00; vld S1, C2, 0x10 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5 + + vld S0, C3, 0x00; vld S1, C3, 0x10 + vfmadd.s D6, S0, VBETA, D6; vfmadd.s D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 + vst D2, C1, 0x00; vst D3, C1, 0x10 + vst D4, C2, 0x00; vst D5, C2, 0x10 + vst D6, C3, 0x00; vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10 + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, X0, 0x04 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2; vfmadd.s D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00; vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2; vfmadd.s D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 + vst D2, C1, 0x00; vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + vld S0, A0, 0x00; vld S1, A0, 0x10 + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADD A0, A0, LDA + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 +.L_M8_END: + PTR_ADDI A, A, 0x20 + PTR_ADDI C, C, 0x20 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_NT_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_NT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_NT_TAIL 1 +.L_M0: + pop_if_used 5, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_tn_lasx.S b/kernel/loongarch64/sgemm_small_kernel_tn_lasx.S new file mode 100644 index 0000000000..e4c8c19519 --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_tn_lasx.S @@ -0,0 +1,764 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M32 $t1 +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 +#define A_cur $s5 +#define TEMP $s6 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define D4 $xr6 +#define D5 $xr7 +#define D6 $xr8 +#define D7 $xr9 +#define D8 $xr10 +#define D9 $xr11 +#define D10 $xr12 +#define D11 $xr13 +#define D12 $xr14 +#define D13 $xr15 +#define D14 $xr16 +#define D15 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + +.macro SGEMM_SMALL_KERNEL_TN_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_TN_M\M\()_N3 +.L_TN_M\M\()_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_TN_M\M\()_N4_END +.L_TN_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + GXOR xv, v, S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; +.endif +.if \M >= 8 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; +.endif + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TN_M\M\()_N4_K1 +.L_TN_M\M\()_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 + GLD xv, , S0, C2, 0x00 + GMADD xvf, s, D2, S0, VBETA, D2 + GLD xv, , S0, C3, 0x00 + GMADD xvf, s, D3, S0, VBETA, D3 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_TN_M\M\()_N4 +.L_TN_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_TN_M\M\()_N1 +.L_TN_M\M\()_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K + bge ZERO, K, .L_TN_M\M\()_N2_END +.L_TN_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + GXOR xv, v, S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; +.endif +.if \M >= 8 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; +.endif + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TN_M\M\()_N2_K1 +.L_TN_M\M\()_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_TN_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_TN_M\M\()_END + GXOR xv, v, D0, D0, D0 + move K1, K + bge ZERO, K, .L_TN_M\M\()_N1_END +.L_TN_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + GXOR xv, v, S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; +.endif +.if \M >= 8 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; +.endif + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TN_M\M\()_N1_K1 +.L_TN_M\M\()_N1_END: + GMUL xvf, s, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00 +.endif +.L_TN_M\M\()_END: +.if \M == 8 + PTR_SLLI TEMP, LDA, 3 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x20 +.elseif \M == 4 + PTR_SLLI TEMP, LDA, 2 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_SLLI TEMP, LDA, 1 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADD A, A, LDA + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 7, 2 + xvreplve0.w VALPHA, VALPHA +#ifndef B0 + xvreplve0.w VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SLLI K4, K, 2 + PTR_SRAI M32, M, 5 + beqz M32, .L_M31 +.L_M32: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M32_N3 +.L_M32_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7, \ + D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \ + D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M32_N4_END +.L_M32_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7, \ + D8, S0, Z2, D8, D9, S1, Z2, D9, D10, S2, Z2, D10, D11, S3, Z2, D11, \ + D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M32_N4_K1 + .L_M32_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA, \ + D8, D8, VALPHA, D9, D9, VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \ + D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60 + GMADD xvf, s, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60 + GMADD xvf, s, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15 +#endif + GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \ + D8, C2, 0x00, D9, C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \ + D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M32_N4 +.L_M32_N3: + andi N2, N, 0x02 + beqz N2, .L_M32_N1 +.L_M32_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K + bge ZERO, K, .L_M32_N2_END +.L_M32_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M32_N2_K1 +.L_M32_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 +#endif + GST xv, , D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M32_N1: + andi N1, N, 0x01 + beqz N1, .L_M32_END + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + bge ZERO, K, .L_M32_N1_END +.L_M32_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M32_N1_K1 +.L_M32_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 2 + PTR_SUB X0, X0, K4 + PTR_ALSL X0, LDB, X0, 2 + move A0, A +.L_M32_END: + PTR_ADDI M32, M32, -1 + PTR_SLLI TEMP, LDA, 5 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x80 + bnez M32, .L_M32 +.L_M31: + andi M16, M, 0x10 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00, Z2, B2, 0x00, Z3, B3, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3, \ + D4, S0, Z2, D4, D5, S1, Z2, D5, \ + D6, S0, Z3, D6, D7, S1, Z3, D7 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20 + GMADD xvf, s, D6, S0, VBETA, D6, D7, S1, VBETA, D7 +#endif + GST xv, , D4, C2, 0x00, D5, C2, 0x20, D6, C3, 0x00, D7, C3, 0x20, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, B1, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20 +.L_M16_END: + PTR_SLLI TEMP, LDA, 4 // 16 * LDA + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x40 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + SGEMM_SMALL_KERNEL_TN_TAIL 8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_TN_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_TN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_TN_TAIL 1 +.L_M0: + pop_if_used 7, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_tn_lsx.S b/kernel/loongarch64/sgemm_small_kernel_tn_lsx.S new file mode 100644 index 0000000000..5fbd399407 --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_tn_lsx.S @@ -0,0 +1,733 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B1 $t6 +#define B2 $t7 +#define B3 $t8 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define A_cur $s5 +#define TEMP $s6 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 + +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro SGEMM_SMALL_KERNEL_TN_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_TN_M\M\()_N3 +.L_TN_M\M\()_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_TN_M\M\()_N4_END +.L_TN_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + vxor.v S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0 +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1 +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3 +.endif + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vldrepl.w Z2, B2, 0x00 + vldrepl.w Z3, B3, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + vfmadd.s D2, S0, Z2, D2 + vfmadd.s D3, S0, Z3, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TN_M\M\()_N4_K1 +.L_TN_M\M\()_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + vld S0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.d FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.s FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.s FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 + vst V2, C2, 0x00 + vst V3, C3, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 + fst.d F2, C2, 0x00 + fst.d F3, C3, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 + fst.s F2, C2, 0x00 + fst.s F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_TN_M\M\()_N4 +.L_TN_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_TN_M\M\()_N1 +.L_TN_M\M\()_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_TN_M\M\()_N2_END +.L_TN_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + vxor.v S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0 +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1 +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3 +.endif + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TN_M\M\()_N2_K1 +.L_TN_M\M\()_N2_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_TN_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_TN_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + bge ZERO, K, .L_TN_M\M\()_N1_END +.L_TN_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + vxor.v S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0 +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1 +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3 +.endif + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TN_M\M\()_N1_K1 +.L_TN_M\M\()_N1_END: + vfmul.s D0, D0, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 +.endif +.L_TN_M\M\()_END: +.if \M == 4 + PTR_SLLI TEMP, LDA, 2 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_SLLI TEMP, LDA, 1 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADD A, A, LDA + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 7, 2 + vreplvei.w VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.w VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SLLI K4, K, 2 + PTR_SRAI M16, M, 4 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + vxor.v D8, D8, D8; vxor.v D9, D9, D9; vxor.v D10, D10, D10; vxor.v D11, D11, D11 + vxor.v D12, D12, D12; vxor.v D13, D13, D13; vxor.v D14, D14, D14; vxor.v D15, D15, D15 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vldrepl.w Z2, B2, 0x00 + vldrepl.w Z3, B3, 0x00 + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + vfmadd.s D4, S0, Z1, D4; vfmadd.s D5, S1, Z1, D5; vfmadd.s D6, S2, Z1, D6; vfmadd.s D7, S3, Z1, D7 + vfmadd.s D8, S0, Z2, D8; vfmadd.s D9, S1, Z2, D9; vfmadd.s D10, S2, Z2, D10; vfmadd.s D11, S3, Z2, D11 + vfmadd.s D12, S0, Z3, D12; vfmadd.s D13, S1, Z3, D13; vfmadd.s D14, S2, Z3, D14; vfmadd.s D15, S3, Z3, D15 + + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA + vfmul.s D8, D8, VALPHA; vfmul.s D9, D9, VALPHA; vfmul.s D10, D10, VALPHA; vfmul.s D11, D11, VALPHA + vfmul.s D12, D12, VALPHA; vfmul.s D13, D13, VALPHA; vfmul.s D14, D14, VALPHA; vfmul.s D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5; vfmadd.s D6, S2, VBETA, D6; vfmadd.s D7, S3, VBETA, D7 + + vld S0, C2, 0x00; vld S1, C2, 0x10; vld S2, C2, 0x20; vld S3, C2, 0x30 + vfmadd.s D8, S0, VBETA, D8; vfmadd.s D9, S1, VBETA, D9; vfmadd.s D10, S2, VBETA, D10; vfmadd.s D11, S3, VBETA, D11 + + vld S0, C3, 0x00; vld S1, C3, 0x10; vld S2, C3, 0x20; vld S3, C3, 0x30 + vfmadd.s D12, S0, VBETA, D12; vfmadd.s D13, S1, VBETA, D13; vfmadd.s D14, S2, VBETA, D14; vfmadd.s D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + vst D8, C2, 0x00; vst D9, C2, 0x10; vst D10, C2, 0x20; vst D11, C2, 0x30 + vst D12, C3, 0x00; vst D13, C3, 0x10; vst D14, C3, 0x20; vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + vfmadd.s D4, S0, Z1, D4; vfmadd.s D5, S1, Z1, D5; vfmadd.s D6, S2, Z1, D6; vfmadd.s D7, S3, Z1, D7 + + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5; vfmadd.s D6, S2, VBETA, D6; vfmadd.s D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_SUB X0, X0, K4 + PTR_ALSL X0, LDB, X0, 2 + move A0, A +.L_M16_END: + PTR_ADDI M16, M16, -1 + PTR_SLLI TEMP, LDA, 4 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x40 + bnez M16, .L_M16 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move X0, B + PTR_ADD B1, X0, LDB + PTR_ADD B2, B1, LDB + PTR_ADD B3, B2, LDB + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, B1, 0x00 + vldrepl.w Z2, B2, 0x00 + vldrepl.w Z3, B3, 0x00 + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2; vfmadd.s D3, S1, Z1, D3 + vfmadd.s D4, S0, Z2, D4; vfmadd.s D5, S1, Z2, D5 + vfmadd.s D6, S0, Z3, D6; vfmadd.s D7, S1, Z3, D7 + + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI B2, B2, 0x04 + PTR_ADDI B3, B3, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00; vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2; vfmadd.s D3, S1, VBETA, D3 + + vld S0, C2, 0x00; vld S1, C2, 0x10 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5 + + vld S0, C3, 0x00; vld S1, C3, 0x10 + vfmadd.s D6, S0, VBETA, D6; vfmadd.s D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 + vst D2, C1, 0x00; vst D3, C1, 0x10 + vst D4, C2, 0x00; vst D5, C2, 0x10 + vst D6, C3, 0x00; vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_SUB B2, B2, K4 + PTR_SUB B3, B3, K4 + PTR_ALSL X0, LDB, X0, 2 + PTR_ALSL B1, LDB, B1, 2 + PTR_ALSL B2, LDB, B2, 2 + PTR_ALSL B3, LDB, B3, 2 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3 + + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, B1, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2; vfmadd.s D3, S1, Z1, D3 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI B1, B1, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00; vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2; vfmadd.s D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 + vst D2, C1, 0x00; vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_SUB X0, X0, K4 + PTR_SUB B1, B1, K4 + PTR_ALSL X0, LDB, X0, 1 + PTR_ALSL B1, LDB, B1, 1 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + PTR_ADDI X0, X0, 0x04 + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 +.L_M8_END: + PTR_SLLI TEMP, LDA, 3 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x20 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_TN_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_TN_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_TN_TAIL 1 +.L_M0: + pop_if_used 7, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_tt_lasx.S b/kernel/loongarch64/sgemm_small_kernel_tt_lasx.S new file mode 100644 index 0000000000..1e1fe57db5 --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_tt_lasx.S @@ -0,0 +1,719 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M32 $t1 +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B_cur $t6 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define A_cur $s5 +#define TEMP $s6 + +#define VALPHA $xr0 +#ifndef B0 +#define VBETA $xr1 +#endif +#define D0 $xr2 +#define D1 $xr3 +#define D2 $xr4 +#define D3 $xr5 +#define D4 $xr6 +#define D5 $xr7 +#define D6 $xr8 +#define D7 $xr9 +#define D8 $xr10 +#define D9 $xr11 +#define D10 $xr12 +#define D11 $xr13 +#define D12 $xr14 +#define D13 $xr15 +#define D14 $xr16 +#define D15 $xr17 +#define S0 $xr18 +#define S1 $xr19 +#define S2 $xr20 +#define S3 $xr21 +#define Z0 $xr22 +#define Z1 $xr23 +#define Z2 $xr24 +#define Z3 $xr25 +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 + +.macro SGEMM_SMALL_KERNEL_TT_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_TT_M\M\()_N3 +.L_TT_M\M\()_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_TT_M\M\()_N4_END +.L_TT_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + GXOR xv, v, S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; +.endif +.if \M >= 8 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; +.endif + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04, Z2, X0, 0x08, Z3, X0, 0x0C + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TT_M\M\()_N4_K1 +.L_TT_M\M\()_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 + GLD xv, , S0, C2, 0x00 + GMADD xvf, s, D2, S0, VBETA, D2 + GLD xv, , S0, C3, 0x00 + GMADD xvf, s, D3, S0, VBETA, D3 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_TT_M\M\()_N4 +.L_TT_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_TT_M\M\()_N1 +.L_TT_M\M\()_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_TT_M\M\()_N2_END +.L_TT_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + GXOR xv, v, S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; +.endif +.if \M >= 8 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; +.endif + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04 + GMADD xvf, s, D0, S0, Z0, D0, D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TT_M\M\()_N2_K1 +.L_TT_M\M\()_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 + GLD xv, , S0, C1, 0x00 + GMADD xvf, s, D1, S0, VBETA, D1 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00, D1, C1, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00, V1, C1, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00, F1, C1, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00, F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_TT_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_TT_M\M\()_END + GXOR xv, v, D0, D0, D0 + move K1, K + move X0, B_cur + bge ZERO, K, .L_TT_M\M\()_N1_END +.L_TT_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + GXOR xv, v, S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; +.endif +.if \M >= 8 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; +.endif + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TT_M\M\()_N1_K1 +.L_TT_M\M\()_N1_END: + GMUL xvf, s, D0, D0, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00 + GMADD xvf, s, D0, S0, VBETA, D0 +#endif +.if \M == 8 + GST xv, , D0, C0, 0x00 +.elseif \M == 4 + GST v, , V0, C0, 0x00 +.elseif \M == 2 + GST f, d, F0, C0, 0x00 +.elseif \M == 1 + GST f, s, F0, C0, 0x00 +.endif +.L_TT_M\M\()_END: +.if \M == 8 + PTR_SLLI TEMP, LDA, 3 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x20 +.elseif \M == 4 + PTR_SLLI TEMP, LDA, 2 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_SLLI TEMP, LDA, 1 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADD A, A, LDA + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 7, 2 + xvreplve0.w VALPHA, VALPHA +#ifndef B0 + xvreplve0.w VBETA, VBETA +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SRAI M32, M, 5 + beqz M32, .L_M31 +.L_M32: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B // Reset B to origin for each M block + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M32_N3 +.L_M32_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7, \ + D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \ + D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M32_N4_END +.L_M32_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04, Z2, X0, 0x08, Z3, X0, 0x0C + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7, \ + D8, S0, Z2, D8, D9, S1, Z2, D9, D10, S2, Z2, D10, D11, S3, Z2, D11, \ + D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M32_N4_K1 + .L_M32_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA, \ + D8, D8, VALPHA, D9, D9, VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \ + D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60 + GMADD xvf, s, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60 + GMADD xvf, s, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15 +#endif + GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \ + D8, C2, 0x00, D9, C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \ + D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M32_N4 +.L_M32_N3: + andi N2, N, 0x02 + beqz N2, .L_M32_N1 +.L_M32_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M32_N2_END +.L_M32_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3, \ + D4, S0, Z1, D4, D5, S1, Z1, D5, D6, S2, Z1, D6, D7, S3, Z1, D7 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M32_N2_K1 +.L_M32_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7 +#endif + GST xv, , D4, C1, 0x00, D5, C1, 0x20, D6, C1, 0x40, D7, C1, 0x60, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M32_N1: + andi N1, N, 0x01 + beqz N1, .L_M32_END + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M32_N1_END +.L_M32_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S2, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S3, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, D2, S2, Z0, D2, D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M32_N1_K1 +.L_M32_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C0, 0x40, D3, C0, 0x60 + PTR_ALSL C0, LDC, C0, 2 + PTR_ADDI B_cur, B_cur, 4 + move A0, A +.L_M32_END: + PTR_ADDI M32, M32, -1 + PTR_SLLI TEMP, LDA, 5 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x80 + bnez M32, .L_M32 +.L_M31: + andi M16, M, 0x10 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \ + D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04, Z2, X0, 0x08, Z3, X0, 0x0C + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3, \ + D4, S0, Z2, D4, D5, S1, Z2, D5, \ + D6, S0, Z3, D6, D7, S1, Z3, D7 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA, \ + D4, D4, VALPHA, D5, D5, VALPHA, D6, D6, VALPHA, D7, D7, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 + GLD xv, , S0, C2, 0x00, S1, C2, 0x20 + GMADD xvf, s, D4, S0, VBETA, D4, D5, S1, VBETA, D5 + GLD xv, , S0, C3, 0x00, S1, C3, 0x20 + GMADD xvf, s, D6, S0, VBETA, D6, D7, S1, VBETA, D7 +#endif + GST xv, , D4, C2, 0x00, D5, C2, 0x20, D6, C3, 0x00, D7, C3, 0x20, \ + D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00, Z1, X0, 0x04 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1, \ + D2, S0, Z1, D2, D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 + GLD xv, , S0, C1, 0x00, S1, C1, 0x20 + GMADD xvf, s, D2, S0, VBETA, D2, D3, S1, VBETA, D3 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20, D2, C1, 0x00, D3, C1, 0x20 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + GXOR xv, v, D0, D0, D0, D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S0, TEMP, 7; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 4; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 5; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 6; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; xvinsgr2vr.w S1, TEMP, 7 + + GLDREPL xv, w, Z0, X0, 0x00 + GMADD xvf, s, D0, S0, Z0, D0, D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + GMUL xvf, s, D0, D0, VALPHA, D1, D1, VALPHA +#ifndef B0 + GLD xv, , S0, C0, 0x00, S1, C0, 0x20 + GMADD xvf, s, D0, S0, VBETA, D0, D1, S1, VBETA, D1 +#endif + GST xv, , D0, C0, 0x00, D1, C0, 0x20 +.L_M16_END: + PTR_SLLI TEMP, LDA, 4 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x40 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + SGEMM_SMALL_KERNEL_TT_TAIL 8 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_TT_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_TT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_TT_TAIL 1 +.L_M0: + pop_if_used 7, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_kernel_tt_lsx.S b/kernel/loongarch64/sgemm_small_kernel_tt_lsx.S new file mode 100644 index 0000000000..6527050fa1 --- /dev/null +++ b/kernel/loongarch64/sgemm_small_kernel_tt_lsx.S @@ -0,0 +1,680 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define ASSEMBLER + +#include "common.h" +#include "loongarch64_asm.S" + +#define M $a0 +#define N $a1 +#define K $a2 +#define A $a3 +#define LDA $a4 +#define ALPHA $f0 +#define B $a5 +#define LDB $a6 +#define C $a7 +#define LDC $t0 +#ifdef B0 +#define BETA $f1 +#endif +#undef ZERO +#define ZERO $r0 + +#define M16 $t1 +#define M8 $t1 +#define M4 $t1 +#define M2 $t1 +#define M1 $t1 +#define N4 $t2 +#define N2 $t2 +#define N1 $t2 +#define K4 $t3 +#define A0 $t4 +#define X0 $t5 +#define B_cur $t6 +#define C0 $s0 +#define C1 $s1 +#define C2 $s2 +#define C3 $s3 +#define K1 $s4 + +#define A_cur $s5 +#define TEMP $s6 + +#define VALPHA $vr0 +#ifndef B0 +#define VBETA $vr1 +#endif +#define D0 $vr2 +#define D1 $vr3 +#define D2 $vr4 +#define D3 $vr5 +#define D4 $vr6 +#define D5 $vr7 +#define D6 $vr8 +#define D7 $vr9 +#define D8 $vr10 +#define D9 $vr11 +#define D10 $vr12 +#define D11 $vr13 +#define D12 $vr14 +#define D13 $vr15 +#define D14 $vr16 +#define D15 $vr17 +#define S0 $vr18 +#define S1 $vr19 +#define S2 $vr20 +#define S3 $vr21 +#define Z0 $vr22 +#define Z1 $vr23 +#define Z2 $vr24 +#define Z3 $vr25 + +#define V0 $vr2 +#define V1 $vr3 +#define V2 $vr4 +#define V3 $vr5 + +#define F0 $f2 +#define F1 $f3 +#define F2 $f4 +#define F3 $f5 +#define FS0 $f18 + +.macro SGEMM_SMALL_KERNEL_TT_TAIL M + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_TT_M\M\()_N3 +.L_TT_M\M\()_N4: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + vxor.v D2, D2, D2 + vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_TT_M\M\()_N4_END +.L_TT_M\M\()_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + vxor.v S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0 +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1 +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3 +.endif + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, X0, 0x04 + vldrepl.w Z2, X0, 0x08 + vldrepl.w Z3, X0, 0x0C + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + vfmadd.s D2, S0, Z2, D2 + vfmadd.s D3, S0, Z3, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TT_M\M\()_N4_K1 +.L_TT_M\M\()_N4_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA + vfmul.s D2, D2, VALPHA + vfmul.s D3, D3, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + vld S0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + vld S0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.d FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.d FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 + fld.s FS0, C2, 0x00; vfmadd.s D2, S0, VBETA, D2 + fld.s FS0, C3, 0x00; vfmadd.s D3, S0, VBETA, D3 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 + vst V2, C2, 0x00 + vst V3, C3, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 + fst.d F2, C2, 0x00 + fst.d F3, C3, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 + fst.s F2, C2, 0x00 + fst.s F3, C3, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_TT_M\M\()_N4 +.L_TT_M\M\()_N3: + andi N2, N, 0x02 + beqz N2, .L_TT_M\M\()_N1 +.L_TT_M\M\()_N2: + vxor.v D0, D0, D0 + vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_TT_M\M\()_N2_END +.L_TT_M\M\()_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + vxor.v S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0 +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1 +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3 +.endif + vldrepl.w Z0, X0, 0x00 + vldrepl.w Z1, X0, 0x04 + vfmadd.s D0, S0, Z0, D0 + vfmadd.s D1, S0, Z1, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TT_M\M\()_N2_K1 +.L_TT_M\M\()_N2_END: + vfmul.s D0, D0, VALPHA + vfmul.s D1, D1, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + vld S0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.d FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 + fld.s FS0, C1, 0x00; vfmadd.s D1, S0, VBETA, D1 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 + vst V1, C1, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 + fst.d F1, C1, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 + fst.s F1, C1, 0x00 +.endif + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_TT_M\M\()_N1: + andi N1, N, 0x01 + beqz N1, .L_TT_M\M\()_END + vxor.v D0, D0, D0 + move K1, K + move X0, B_cur + bge ZERO, K, .L_TT_M\M\()_N1_END +.L_TT_M\M\()_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + vxor.v S0, S0, S0 +.if \M >= 1 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0 +.endif +.if \M >= 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1 +.endif +.if \M >= 4 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2 + PTR_ADD A_cur, A_cur, LDA; ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3 +.endif + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_TT_M\M\()_N1_K1 +.L_TT_M\M\()_N1_END: + vfmul.s D0, D0, VALPHA +#ifndef B0 +.if \M == 4 + vld S0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 2 + fld.d FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.elseif \M == 1 + fld.s FS0, C0, 0x00; vfmadd.s D0, S0, VBETA, D0 +.endif +#endif +.if \M == 4 + vst V0, C0, 0x00 +.elseif \M == 2 + fst.d F0, C0, 0x00 +.elseif \M == 1 + fst.s F0, C0, 0x00 +.endif +.L_TT_M\M\()_END: +.if \M == 4 + PTR_SLLI TEMP, LDA, 2 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x10 +.elseif \M == 2 + PTR_SLLI TEMP, LDA, 1 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x08 +.elseif \M == 1 + PTR_ADD A, A, LDA + PTR_ADDI C, C, 0x04 +.endif +.endm + + PROLOGUE + PTR_LD LDC, $sp, 0 + push_if_used 7, 2 + vreplvei.w VALPHA, VALPHA, 0 +#ifndef B0 + vreplvei.w VBETA, VBETA, 0 +#endif + PTR_SLLI LDA, LDA, 2 + PTR_SLLI LDB, LDB, 2 + PTR_SLLI LDC, LDC, 2 + PTR_SRAI M16, M, 4 + beqz M16, .L_M15 +.L_M16: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M16_N3 +.L_M16_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + vxor.v D8, D8, D8; vxor.v D9, D9, D9; vxor.v D10, D10, D10; vxor.v D11, D11, D11 + vxor.v D12, D12, D12; vxor.v D13, D13, D13; vxor.v D14, D14, D14; vxor.v D15, D15, D15 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M16_N4_END +.L_M16_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 3 + + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, X0, 0x04; vldrepl.w Z2, X0, 0x08; vldrepl.w Z3, X0, 0x0C + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + vfmadd.s D4, S0, Z1, D4; vfmadd.s D5, S1, Z1, D5; vfmadd.s D6, S2, Z1, D6; vfmadd.s D7, S3, Z1, D7 + vfmadd.s D8, S0, Z2, D8; vfmadd.s D9, S1, Z2, D9; vfmadd.s D10, S2, Z2, D10; vfmadd.s D11, S3, Z2, D11 + vfmadd.s D12, S0, Z3, D12; vfmadd.s D13, S1, Z3, D13; vfmadd.s D14, S2, Z3, D14; vfmadd.s D15, S3, Z3, D15 + + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N4_K1 +.L_M16_N4_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA + vfmul.s D8, D8, VALPHA; vfmul.s D9, D9, VALPHA; vfmul.s D10, D10, VALPHA; vfmul.s D11, D11, VALPHA + vfmul.s D12, D12, VALPHA; vfmul.s D13, D13, VALPHA; vfmul.s D14, D14, VALPHA; vfmul.s D15, D15, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5; vfmadd.s D6, S2, VBETA, D6; vfmadd.s D7, S3, VBETA, D7 + + vld S0, C2, 0x00; vld S1, C2, 0x10; vld S2, C2, 0x20; vld S3, C2, 0x30 + vfmadd.s D8, S0, VBETA, D8; vfmadd.s D9, S1, VBETA, D9; vfmadd.s D10, S2, VBETA, D10; vfmadd.s D11, S3, VBETA, D11 + + vld S0, C3, 0x00; vld S1, C3, 0x10; vld S2, C3, 0x20; vld S3, C3, 0x30 + vfmadd.s D12, S0, VBETA, D12; vfmadd.s D13, S1, VBETA, D13; vfmadd.s D14, S2, VBETA, D14; vfmadd.s D15, S3, VBETA, D15 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + vst D8, C2, 0x00; vst D9, C2, 0x10; vst D10, C2, 0x20; vst D11, C2, 0x30 + vst D12, C3, 0x00; vst D13, C3, 0x10; vst D14, C3, 0x20; vst D15, C3, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M16_N4 +.L_M16_N3: + andi N2, N, 0x02 + beqz N2, .L_M16_N1 +.L_M16_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N2_END +.L_M16_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 3 + + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, X0, 0x04 + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + vfmadd.s D4, S0, Z1, D4; vfmadd.s D5, S1, Z1, D5; vfmadd.s D6, S2, Z1, D6; vfmadd.s D7, S3, Z1, D7 + + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N2_K1 +.L_M16_N2_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 + + vld S0, C1, 0x00; vld S1, C1, 0x10; vld S2, C1, 0x20; vld S3, C1, 0x30 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5; vfmadd.s D6, S2, VBETA, D6; vfmadd.s D7, S3, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + vst D4, C1, 0x00; vst D5, C1, 0x10; vst D6, C1, 0x20; vst D7, C1, 0x30 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M16_N1: + andi N1, N, 0x01 + beqz N1, .L_M16_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M16_N1_END +.L_M16_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S2, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S3, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1; vfmadd.s D2, S2, Z0, D2; vfmadd.s D3, S3, Z0, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M16_N1_K1 +.L_M16_N1_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10; vld S2, C0, 0x20; vld S3, C0, 0x30 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1; vfmadd.s D2, S2, VBETA, D2; vfmadd.s D3, S3, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10; vst D2, C0, 0x20; vst D3, C0, 0x30 + PTR_ALSL C0, LDC, C0, 2 + PTR_ADDI B_cur, B_cur, 4 + move A0, A +.L_M16_END: + PTR_ADDI M16, M16, -1 + PTR_SLLI TEMP, LDA, 4 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x40 + bnez M16, .L_M16 +.L_M15: + andi M8, M, 0x08 + beqz M8, .L_M7 +.L_M8: + PTR_SRAI N4, N, 2 + move A0, A + move B_cur, B + move C0, C + PTR_ADD C1, C0, LDC + PTR_ADD C2, C1, LDC + PTR_ADD C3, C2, LDC + beqz N4, .L_M8_N3 +.L_M8_N4: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + vxor.v D4, D4, D4; vxor.v D5, D5, D5; vxor.v D6, D6, D6; vxor.v D7, D7, D7 + move K1, K + move X0, B_cur + PTR_ADDI N4, N4, -1 + bge ZERO, K, .L_M8_N4_END +.L_M8_N4_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3 + + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, X0, 0x04; vldrepl.w Z2, X0, 0x08; vldrepl.w Z3, X0, 0x0C + + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2; vfmadd.s D3, S1, Z1, D3 + vfmadd.s D4, S0, Z2, D4; vfmadd.s D5, S1, Z2, D5 + vfmadd.s D6, S0, Z3, D6; vfmadd.s D7, S1, Z3, D7 + + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M8_N4_K1 +.L_M8_N4_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA + vfmul.s D4, D4, VALPHA; vfmul.s D5, D5, VALPHA; vfmul.s D6, D6, VALPHA; vfmul.s D7, D7, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00; vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2; vfmadd.s D3, S1, VBETA, D3 + + vld S0, C2, 0x00; vld S1, C2, 0x10 + vfmadd.s D4, S0, VBETA, D4; vfmadd.s D5, S1, VBETA, D5 + + vld S0, C3, 0x00; vld S1, C3, 0x10 + vfmadd.s D6, S0, VBETA, D6; vfmadd.s D7, S1, VBETA, D7 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 + vst D2, C1, 0x00; vst D3, C1, 0x10 + vst D4, C2, 0x00; vst D5, C2, 0x10 + vst D6, C3, 0x00; vst D7, C3, 0x10 + PTR_ALSL C0, LDC, C0, 2 + PTR_ALSL C1, LDC, C1, 2 + PTR_ALSL C2, LDC, C2, 2 + PTR_ALSL C3, LDC, C3, 2 + PTR_ADDI B_cur, B_cur, 16 + move A0, A + bnez N4, .L_M8_N4 +.L_M8_N3: + andi N2, N, 0x02 + beqz N2, .L_M8_N1 +.L_M8_N2: + vxor.v D0, D0, D0; vxor.v D1, D1, D1; vxor.v D2, D2, D2; vxor.v D3, D3, D3 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N2_END +.L_M8_N2_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3 + + vldrepl.w Z0, X0, 0x00; vldrepl.w Z1, X0, 0x04 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + vfmadd.s D2, S0, Z1, D2; vfmadd.s D3, S1, Z1, D3 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M8_N2_K1 +.L_M8_N2_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA; vfmul.s D2, D2, VALPHA; vfmul.s D3, D3, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 + + vld S0, C1, 0x00; vld S1, C1, 0x10 + vfmadd.s D2, S0, VBETA, D2; vfmadd.s D3, S1, VBETA, D3 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 + vst D2, C1, 0x00; vst D3, C1, 0x10 + PTR_ALSL C0, LDC, C0, 1 + PTR_ALSL C1, LDC, C1, 1 + PTR_ADDI B_cur, B_cur, 8 + move A0, A +.L_M8_N1: + andi N1, N, 0x01 + beqz N1, .L_M8_END + vxor.v D0, D0, D0; vxor.v D1, D1, D1 + move K1, K + move X0, B_cur + bge ZERO, K, .L_M8_N1_END +.L_M8_N1_K1: + PTR_ADDI K1, K1, -1 + move A_cur, A0 + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S0, TEMP, 3; PTR_ADD A_cur, A_cur, LDA + + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 0; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 1; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 2; PTR_ADD A_cur, A_cur, LDA + ld.w TEMP, A_cur, 0; vinsgr2vr.w S1, TEMP, 3 + + vldrepl.w Z0, X0, 0x00 + vfmadd.s D0, S0, Z0, D0; vfmadd.s D1, S1, Z0, D1 + PTR_ADD X0, X0, LDB + PTR_ADDI A0, A0, 0x04 + bnez K1, .L_M8_N1_K1 +.L_M8_N1_END: + vfmul.s D0, D0, VALPHA; vfmul.s D1, D1, VALPHA +#ifndef B0 + vld S0, C0, 0x00; vld S1, C0, 0x10 + vfmadd.s D0, S0, VBETA, D0; vfmadd.s D1, S1, VBETA, D1 +#endif + vst D0, C0, 0x00; vst D1, C0, 0x10 +.L_M8_END: + PTR_SLLI TEMP, LDA, 3 + PTR_ADD A, A, TEMP + PTR_ADDI C, C, 0x20 +.L_M7: + andi M4, M, 0x04 + beqz M4, .L_M3 +.L_M4: + SGEMM_SMALL_KERNEL_TT_TAIL 4 +.L_M3: + andi M2, M, 0x02 + beqz M2, .L_M1 +.L_M2: + SGEMM_SMALL_KERNEL_TT_TAIL 2 +.L_M1: + andi M1, M, 0x01 + beqz M1, .L_M0 + SGEMM_SMALL_KERNEL_TT_TAIL 1 +.L_M0: + pop_if_used 7, 2 + jirl $r0, $r1, 0x0 + EPILOGUE \ No newline at end of file diff --git a/kernel/loongarch64/sgemm_small_matrix_permit.c b/kernel/loongarch64/sgemm_small_matrix_permit.c new file mode 100644 index 0000000000..bc215dcd49 --- /dev/null +++ b/kernel/loongarch64/sgemm_small_matrix_permit.c @@ -0,0 +1,56 @@ +/*************************************************************************** +Copyright (c) 2026, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta) +{ + double MNK = (double) M * (double) N * (double) K; + +#ifdef LA464 + if (transa) { + if (MNK <= 14.0 * 14.0 * 14.0) + return 1; + } else { + if (MNK <= 80.0 * 80.0 * 80.0) + return 1; + } +#endif + +#ifdef LA264 + if (transa) { + if (MNK <= 80.0 * 80.0 * 80.0) + return 1; + } else { + if (MNK <= 100.0 * 100.0 * 100.0) + return 1; + } +#endif + + return 0; +} +